{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:11.250979Z",
     "start_time": "2025-01-27T04:36:04.863049Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:45.976903Z",
     "iopub.status.busy": "2025-01-27T13:00:45.976708Z",
     "iopub.status.idle": "2025-01-27T13:00:48.320487Z",
     "shell.execute_reply": "2025-01-27T13:00:48.319894Z",
     "shell.execute_reply.started": "2025-01-27T13:00:45.976879Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=10, micro=14, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cu124\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd1ebc34a8798df6",
   "metadata": {},
   "source": [
    "# 数据加载与处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "939ac15db682d46d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:11.397006Z",
     "start_time": "2025-01-27T04:36:11.251974Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:48.322209Z",
     "iopub.status.busy": "2025-01-27T13:00:48.321798Z",
     "iopub.status.idle": "2025-01-27T13:00:48.356226Z",
     "shell.execute_reply": "2025-01-27T13:00:48.355683Z",
     "shell.execute_reply.started": "2025-01-27T13:00:48.322183Z"
    }
   },
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# 因为西班牙语有一些是特殊字符，所以我们需要unicode转ascii，\n",
    "# 这样值变小了，因为unicode太大\n",
    "# 将 Unicode 字符串转换为 ASCII 字符串，去除其中的重音符号\n",
    "def unicode_to_ascii(s):\n",
    "    # NFD是转换方法，把每一个字节拆开，Mn是重音，所以去除\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "# #下面我们找个样本测试一下\n",
    "# # 加u代表对字符串进行unicode编码\n",
    "# en_sentence = u\"May I borrow this book?\"\n",
    "# sp_sentence = u\"¿Puedo tomar prestado este libro?\"\n",
    "# \n",
    "# print(unicode_to_ascii(en_sentence))\n",
    "# print(unicode_to_ascii(sp_sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f631de2ef6279160",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:11.403028Z",
     "start_time": "2025-01-27T04:36:11.398004Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:48.357160Z",
     "iopub.status.busy": "2025-01-27T13:00:48.356865Z",
     "iopub.status.idle": "2025-01-27T13:00:48.361895Z",
     "shell.execute_reply": "2025-01-27T13:00:48.361123Z",
     "shell.execute_reply.started": "2025-01-27T13:00:48.357138Z"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess_sentence(w):\n",
    "    # 变为小写，去掉多余的空格，变成小写，id少一些\n",
    "    w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "    # 特定标点符号（?.!,¿）前后添加空格\n",
    "    # eg: \"he is a boy.\" => \"he is a boy . \"\n",
    "    # 这样可以使得模型更容易分辨单词和标点符号\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    # 因为可能有多余空格，替换为一个空格，所以处理一下\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    # 将字符串 w 中所有非字母字符（包括标点符号 ?.!,¿）替换为空格\n",
    "    # [^...] 表示匹配不在括号内的字符。\n",
    "    # a-zA-Z 匹配所有大小写字母。\n",
    "    # ?.!,¿ 匹配特定的标点符号。\n",
    "    #+ 表示匹配一个或多个连续字符。\n",
    "    w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "\n",
    "    #  先去除末尾空白字符，再去除首尾空白字符。\n",
    "    w = w.rstrip().strip()\n",
    "    return w\n",
    "\n",
    "# print(preprocess_sentence(en_sentence))\n",
    "# print(preprocess_sentence(sp_sentence))\n",
    "# print(preprocess_sentence(sp_sentence).encode('utf-8'))  #¿是占用两个字节的"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c36ecfc4cba8609b",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c78b45975b61bd8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:11.409030Z",
     "start_time": "2025-01-27T04:36:11.405541Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:48.362738Z",
     "iopub.status.busy": "2025-01-27T13:00:48.362551Z",
     "iopub.status.idle": "2025-01-27T13:00:48.365652Z",
     "shell.execute_reply": "2025-01-27T13:00:48.364932Z",
     "shell.execute_reply.started": "2025-01-27T13:00:48.362718Z"
    }
   },
   "outputs": [],
   "source": [
    "# #zip例子\n",
    "# a = [[1, 2], [4, 5], [7, 8]]\n",
    "# # *a 是 Python 的解包操作符，将 a 解包为 3 个子列表：\n",
    "# # zip 函数将解包后的子列表按位置组合\n",
    "# zipped = list(zip(*a))\n",
    "# print(zipped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f0e1f6b9133f7cc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:11.414026Z",
     "start_time": "2025-01-27T04:36:11.410026Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:48.366411Z",
     "iopub.status.busy": "2025-01-27T13:00:48.366235Z",
     "iopub.status.idle": "2025-01-27T13:00:48.369244Z",
     "shell.execute_reply": "2025-01-27T13:00:48.368595Z",
     "shell.execute_reply.started": "2025-01-27T13:00:48.366392Z"
    }
   },
   "outputs": [],
   "source": [
    "# split_index1 = np.random.choice(a=[\"train\", \"test\"],\n",
    "#                                 replace=True, p=[0.9, 0.1], size=100)\n",
    "# print(len(split_index1))\n",
    "# print(\"-\" * 50)\n",
    "# split_index1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4bc7f9f4cb98ac0f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:11.878382Z",
     "start_time": "2025-01-27T04:36:11.415027Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:48.369960Z",
     "iopub.status.busy": "2025-01-27T13:00:48.369778Z",
     "iopub.status.idle": "2025-01-27T13:00:48.731336Z",
     "shell.execute_reply": "2025-01-27T13:00:48.730763Z",
     "shell.execute_reply.started": "2025-01-27T13:00:48.369933Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "class LangPairDataset(Dataset):\n",
    "    fpath = Path(r\"./data_spa_en/spa.txt\")  #数据文件路径\n",
    "    cache_path = Path(r\"./.cache/lang_pair.npy\")  #缓存文件路径 \n",
    "    # 按照9:1划分训练集和测试集\n",
    "    split_index = np.random.choice(a=[\"train\", \"test\"], replace=True, p=[0.9, 0.1], size=118964)\n",
    "\n",
    "    def __init__(self, mode=\"train\", cache=False):\n",
    "        # if cache or not self.cache_path.exists() 表示：\n",
    "        # 如果启用缓存（cache 为 True），或者缓存文件不存在，则条件成立。\n",
    "        if cache or not self.cache_path.exists():\n",
    "            # parent 属性返回 self.cache_path 的父目录。\n",
    "            # mkdir 是 Path 对象的方法，用于创建目录。\n",
    "            # parents=True 表示递归创建父目录（如果父目录不存在）。\n",
    "            # exist_ok=True 表示如果目录已存在，不会抛出错误。\n",
    "            self.cache_path.parent.mkdir(exist_ok=True, parents=True)\n",
    "            with open(self.fpath, \"r\", encoding=\"utf-8\") as fd:\n",
    "                # 从文件中读取所有行，并将其存储在一个列表中\n",
    "                lines = fd.readlines()\n",
    "                # 将 lines 中的每一行按制表符 \\t 分割，并对每个分割后的部分调用 preprocess_sentence 函数进行预处理\n",
    "                lang_pair = [[preprocess_sentence(w) for w in l.split(\"\\t\")] for l in lines]\n",
    "                # 分离出目标语言和源语言\n",
    "                trg, src = zip(*lang_pair)\n",
    "                trg = np.array(trg)\n",
    "                src = np.array(src)\n",
    "                # 保存为npy文件,方便下次直接读取,不用再处理\n",
    "                np.save(self.cache_path, {\"trg\": trg, \"src\": src})\n",
    "        else:\n",
    "            # 从 .npy 文件中加载数据，并将其转换为 Python 字典（或其他对象）\n",
    "            # np.load 是 NumPy 提供的函数，用于从 .npy 或 .npz 文件中加载数据。\n",
    "            # allow_pickle=True 表示允许从 .npy 文件中加载 Python 对象。\n",
    "            # .item() 将其转换为 Python 对象（如字典、列表、字典等）\n",
    "            lang_pair = np.load(self.cache_path, allow_pickle=True).item()\n",
    "            trg = lang_pair[\"trg\"]\n",
    "            src = lang_pair[\"src\"]\n",
    "\n",
    "        # 按照index拿到训练集的 标签语言 --英语\n",
    "        self.trg = trg[self.split_index == mode]\n",
    "        # 按照index拿到训练集的源语言 --西班牙\n",
    "        self.src = src[self.split_index == mode]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self.src[index], self.trg[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.src)\n",
    "\n",
    "\n",
    "train_ds = LangPairDataset(mode=\"train\")\n",
    "test_ds = LangPairDataset(mode=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83a0eec9d0d9e91d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:11.884379Z",
     "start_time": "2025-01-27T04:36:11.880380Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:48.733620Z",
     "iopub.status.busy": "2025-01-27T13:00:48.733327Z",
     "iopub.status.idle": "2025-01-27T13:00:48.737431Z",
     "shell.execute_reply": "2025-01-27T13:00:48.736694Z",
     "shell.execute_reply.started": "2025-01-27T13:00:48.733597Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source:una huella de carbono es la cantidad de contaminacion de dioxido de carbono que producimos como producto de nuestras actividades . algunas personas intentan reducir su huella de carbono porque estan preocupados acerca del cambio climatico .\n",
      "target:a carbon footprint is the amount of carbon dioxide pollution that we produce as a result of our activities . some people try to reduce their carbon footprint because they are concerned about climate change .\n"
     ]
    }
   ],
   "source": [
    "print(\"source:{}\\ntarget:{}\".format(*train_ds[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "906e6d420362b515",
   "metadata": {},
   "source": [
    "## Tokenizer\n",
    "\n",
    "这里有两种处理方式，分别对应着 encoder 和 decoder 的 word embedding 是否共享，这里实现不共享的方案。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ae72149b871c746d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.585464Z",
     "start_time": "2025-01-27T04:36:11.886379Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:48.738408Z",
     "iopub.status.busy": "2025-01-27T13:00:48.738084Z",
     "iopub.status.idle": "2025-01-27T13:00:49.347233Z",
     "shell.execute_reply": "2025-01-27T13:00:49.346612Z",
     "shell.execute_reply.started": "2025-01-27T13:00:48.738386Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "\n",
    "def get_word_idx(ds, mode=\"src\", threshold=2):\n",
    "    #载入词表，看下词表长度，词表就像英语字典\n",
    "    word2idx = {\n",
    "        \"[PAD]\": 0,  # 填充 token\n",
    "        \"[BOS]\": 1,  # begin of sentence\n",
    "        \"[UNK]\": 2,  # 未知 token\n",
    "        \"[EOS]\": 3,  # end of sentence\n",
    "    }\n",
    "    idx2word = {value: key for key, value in word2idx.items()}\n",
    "    index = len(idx2word)\n",
    "    # 出现次数低于此的token舍弃\n",
    "    threshold = 1\n",
    "    # 如果数据集有很多个G，那是用for循环的，不能' '.join\n",
    "    word_list = \" \".join([pair[0 if mode == \"src\" else 1] for pair in ds]).split()\n",
    "    # print(type(word_list)) # <class 'list'>\n",
    "    # 统计词频,counter类似字典，key是单词，value是出现次数\n",
    "    counter = Counter(word_list)\n",
    "    # print(\"word count:\", len(counter))\n",
    "\n",
    "    for token, count in counter.items():\n",
    "        # 出现次数大于阈值的token加入词表\n",
    "        if count >= threshold:\n",
    "            word2idx[token] = index  # 加入词表\n",
    "            idx2word[index] = token  # 加入反向词典\n",
    "            index += 1\n",
    "\n",
    "    return word2idx, idx2word\n",
    "\n",
    "\n",
    "# 源语言词表  西班牙语\n",
    "src_word2idx, src_idx2word = get_word_idx(train_ds, mode=\"src\")\n",
    "# 目标语言词表 英语\n",
    "trg_word2idx, trg_idx2word = get_word_idx(train_ds, mode=\"trg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1585f8ee2b3174b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.598464Z",
     "start_time": "2025-01-27T04:36:12.587462Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.348121Z",
     "iopub.status.busy": "2025-01-27T13:00:49.347919Z",
     "iopub.status.idle": "2025-01-27T13:00:49.357864Z",
     "shell.execute_reply": "2025-01-27T13:00:49.357153Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.348101Z"
    }
   },
   "outputs": [],
   "source": [
    "class Tokenizer:\n",
    "    def __init__(self, word2idx, idx2word, max_length=500,\n",
    "                 pad_idx=0, bos_idx=1, eos_idx=3, unk_idx=2):\n",
    "        self.word2idx = word2idx\n",
    "        self.idx2word = idx2word\n",
    "        self.max_length = max_length\n",
    "        self.pad_idx = pad_idx\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.unk_idx = unk_idx\n",
    "\n",
    "    def encode(self, text_list, padding_first=False, add_bos=True, add_eos=True,\n",
    "               return_mask=False):\n",
    "        # 如果padding_first == True，则padding加载前面，否则加载后面\n",
    "        # return_mask: 是否返回mask(掩码），mask用于指示哪些是padding的，哪些是真实的token \n",
    "        # 若启动bos_eos，则在句首和句尾分别添加bos和eos，则 add_bos=True, add_eos=True即都为1\n",
    "        max_length = min(self.max_length, add_bos + add_eos + max([len(text) for text in text_list]))\n",
    "        indices_list = []\n",
    "        for text in text_list:\n",
    "            # 如果词表中没有这个词，就用unk_idx代替，indices是一个list,里面是每个词的index,也就是一个样本的index\n",
    "            indices = [self.word2idx.get(word, self.unk_idx) for word in text[:max_length - add_eos - add_bos]]\n",
    "            if add_bos:\n",
    "                indices = [self.bos_idx] + indices\n",
    "            if add_eos:\n",
    "                indices = indices + [self.eos_idx]\n",
    "            if padding_first:  # padding加载前面，超参可以调\n",
    "                indices = [self.pad_idx] * (max_length - len(indices)) + indices\n",
    "            else:  # padding加载后面\n",
    "                indices = indices + [self.pad_idx] * (max_length - len(indices))\n",
    "            indices_list.append(indices)\n",
    "        input_ids = torch.tensor(indices_list)  # 转换为tensor\n",
    "        # mask是一个和input_ids一样大小的tensor，0代表token，1代表padding，mask用于去除padding的影响\n",
    "        masks = (input_ids == self.pad_idx).to(dtype=torch.float64)\n",
    "        return input_ids if not return_mask else (input_ids, masks)\n",
    "\n",
    "    def decode(self, indices_list, remove_bos=True, remove_eos=True, remove_pad=True, split=False):\n",
    "        text_list = []\n",
    "        for indices in indices_list:\n",
    "            text = []\n",
    "            for index in indices:\n",
    "                # 如果词表中没有这个词，就用unk_idx代替\n",
    "                word = self.idx2word.get(index, \"[UNK]\")\n",
    "                if remove_bos and word == \"[BOS]\":\n",
    "                    continue\n",
    "                if remove_eos and word == \"[EOS]\":  # 如果到达eos，就结束\n",
    "                    break\n",
    "                if remove_pad and word == \"[PAD]\":  # 如果到达pad，就结束\n",
    "                    break\n",
    "                text.append(word)  # 单词添加到列表中\n",
    "            # 把列表中的单词拼接，变为一个句子\n",
    "            text_list.append(\" \".join(text) if not split else text)\n",
    "        return text_list\n",
    "\n",
    "\n",
    "#两个相对于1个toknizer的好处是embedding的参数量减少\n",
    "# 源语言tokenizer\n",
    "src_tokenizer = Tokenizer(word2idx=src_word2idx, idx2word=src_idx2word)\n",
    "# 目标语言tokenizer\n",
    "trg_tokenizer = Tokenizer(word2idx=trg_word2idx, idx2word=trg_idx2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "602e9cd5a872b5a5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.606863Z",
     "start_time": "2025-01-27T04:36:12.602464Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.358751Z",
     "iopub.status.busy": "2025-01-27T13:00:49.358565Z",
     "iopub.status.idle": "2025-01-27T13:00:49.361688Z",
     "shell.execute_reply": "2025-01-27T13:00:49.361186Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.358731Z"
    }
   },
   "outputs": [],
   "source": [
    "# # trg_tokenizer.encode([[\"hello\"], [\"hello\", \"world\"]], add_bos=True, add_eos=False,return_mask=True)\n",
    "# raw_text = [\"hello world\".split(), \"tokenize text datas with batch\".split(), \"this is a test\".split()]\n",
    "# indices,mask = trg_tokenizer.encode(raw_text, padding_first=False, add_bos=True, add_eos=True,return_mask=True)\n",
    "# \n",
    "# print(\"raw text\"+'-'*10)\n",
    "# for raw in raw_text:\n",
    "#     print(raw)\n",
    "# print(\"mask\"+'-'*10)\n",
    "# for m in mask:\n",
    "#     print(m)\n",
    "# print(\"indices\"+'-'*10)\n",
    "# for index in indices:\n",
    "#     print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1a5102956b2ec3f1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.613889Z",
     "start_time": "2025-01-27T04:36:12.608855Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.362736Z",
     "iopub.status.busy": "2025-01-27T13:00:49.362316Z",
     "iopub.status.idle": "2025-01-27T13:00:49.364969Z",
     "shell.execute_reply": "2025-01-27T13:00:49.364484Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.362715Z"
    }
   },
   "outputs": [],
   "source": [
    "# decode_text = trg_tokenizer.decode(indices.tolist(), remove_bos=False, remove_eos=False, remove_pad=False)\n",
    "# print(\"decode text\"+'-'*10)\n",
    "# for decode in decode_text:\n",
    "#     print(decode)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df29f25d3c740e46",
   "metadata": {},
   "source": [
    "## DataLoader\n",
    "\n",
    "encoder设置mask的原因：在softmax计算时，padding的位置计算结果趋近于0\n",
    "\n",
    "decoder设置mask的原因：在计算loss时，padding的位置的loss不参与计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "caca3658e2a7a115",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.624109Z",
     "start_time": "2025-01-27T04:36:12.616885Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.365867Z",
     "iopub.status.busy": "2025-01-27T13:00:49.365563Z",
     "iopub.status.idle": "2025-01-27T13:00:49.370921Z",
     "shell.execute_reply": "2025-01-27T13:00:49.370402Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.365848Z"
    }
   },
   "outputs": [],
   "source": [
    "def collate_fn(batch):\n",
    "    # 取batch内第0列进行分词，赋给src_words\n",
    "    src_words = [pair[0].split() for pair in batch]\n",
    "    # 取batch内第1列进行分词，赋给trg_words\n",
    "    trg_words = [pair[1].split() for pair in batch]\n",
    "\n",
    "    # [PAD] [BOS] src [EOS]\n",
    "    encoder_inputs, encoder_inputs_mask = src_tokenizer.encode(src_words, padding_first=True, add_bos=True,\n",
    "                                                               add_eos=True, return_mask=True)\n",
    "\n",
    "    # 目标语言 [BOS] trg [PAD]\n",
    "    decoder_inputs = trg_tokenizer.encode(trg_words, padding_first=False, add_bos=True, add_eos=False,\n",
    "                                          return_mask=False)\n",
    "\n",
    "    # 用于后续计算loss\n",
    "    # trg [EOS] [PAD]\n",
    "    decoder_labels, decoder_labels_mask = trg_tokenizer.encode(trg_words, padding_first=False, add_bos=False,\n",
    "                                                               add_eos=True, return_mask=True)\n",
    "\n",
    "    return {\n",
    "        \"encoder_inputs\": encoder_inputs.to(device),\n",
    "        \"encoder_inputs_mask\": encoder_inputs_mask.to(device),\n",
    "        \"decoder_inputs\": decoder_inputs.to(device),\n",
    "        \"decoder_labels\": decoder_labels.to(device),\n",
    "        # mask用于去除padding的影响，计算loss时用\n",
    "        \"decoder_labels_mask\": decoder_labels_mask.to(device),\n",
    "    }  # 当返回的数据较多时，用dict返回比较合理\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "252c37d675f553a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.630106Z",
     "start_time": "2025-01-27T04:36:12.626107Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.371960Z",
     "iopub.status.busy": "2025-01-27T13:00:49.371654Z",
     "iopub.status.idle": "2025-01-27T13:00:49.374261Z",
     "shell.execute_reply": "2025-01-27T13:00:49.373791Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.371940Z"
    }
   },
   "outputs": [],
   "source": [
    "# for batch in train_loader:\n",
    "#     for key, value in batch.items():\n",
    "#         print(key)\n",
    "#         print(value.shape)\n",
    "#         print(\"-\"*50)\n",
    "#     break    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "929678e26f67eccc",
   "metadata": {},
   "source": [
    "# 定义模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19fb8fb3385416ab",
   "metadata": {},
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "998ca06a810799f6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.638496Z",
     "start_time": "2025-01-27T04:36:12.632106Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.375224Z",
     "iopub.status.busy": "2025-01-27T13:00:49.374899Z",
     "iopub.status.idle": "2025-01-27T13:00:49.379248Z",
     "shell.execute_reply": "2025-01-27T13:00:49.378778Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.375203Z"
    }
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024, num_layers=1):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "\n",
    "    def forward(self, encoder_inputs):\n",
    "        # encoder_inputs.shape = [batch size, sequence_length]\n",
    "        embeds = self.embedding(encoder_inputs)\n",
    "        # embeds.shape = [batch size, sequence_length, embedding_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, sequence_length, hidden_dim]\n",
    "        # hidden.shape = [num_layers, batch size, hidden_dim]\n",
    "        return seq_output, hidden\n",
    "        # encoder_outputs[:,-1,:]==hidden[-1,:,:]  全是True 说明输出和hidden是相同的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4862d8fe85734897",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.645063Z",
     "start_time": "2025-01-27T04:36:12.640494Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.380221Z",
     "iopub.status.busy": "2025-01-27T13:00:49.379854Z",
     "iopub.status.idle": "2025-01-27T13:00:49.382901Z",
     "shell.execute_reply": "2025-01-27T13:00:49.382379Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.380199Z"
    }
   },
   "outputs": [],
   "source": [
    "# #把上面的Encoder写一个例子，看看输出的shape\n",
    "# encoder = Encoder(vocab_size=100, embedding_dim=256, hidden_dim=1024, num_layers=4)\n",
    "# # 生成一个形状为 (2, 50) 的随机整数张量，其中每个元素的取值范围是 [0, 100)\n",
    "# encoder_inputs = torch.randint(0, 100, (2, 50)) # [2,50]\n",
    "# encoder_outputs, hidden = encoder(encoder_inputs)\n",
    "# print(encoder_outputs.shape) # [2,50,1024]\n",
    "# print(hidden.shape) # [4,2,1024]\n",
    "# print(encoder_outputs[:,-1,:]) # 取最后一个时间步的输出\n",
    "# print(hidden[-1,:,:]) #取最后一层的hidden\n",
    "# print(encoder_outputs[:,-1,:]==hidden[-1,:,:]) # 全是True 说明输出和hidden是相同的"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96f2a739975ee635",
   "metadata": {},
   "source": [
    "## BahdanauAttention公式\n",
    "score = FC(tanh(FC(EO) + FC(H))) 其中FC(EO)的FC是Wk,FC(H)的FC是Wq,最外面的FC是V \n",
    "\n",
    "attention_weights = softmax(score, axis = 1)  \n",
    "\n",
    "context = sum(attention_weights * EO, axis = 1) #对EO做加权求和，得到上下文向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "df0075e30e05dd4a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.655977Z",
     "start_time": "2025-01-27T04:36:12.648060Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.383856Z",
     "iopub.status.busy": "2025-01-27T13:00:49.383590Z",
     "iopub.status.idle": "2025-01-27T13:00:49.389664Z",
     "shell.execute_reply": "2025-01-27T13:00:49.389117Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.383833Z"
    }
   },
   "outputs": [],
   "source": [
    "class BahdanauAttention(nn.Module):\n",
    "    def __init__(self, hidden_dim=1024):\n",
    "        super().__init__()\n",
    "        # 对keys做运算，encoder的输出EO\n",
    "        self.Wk = nn.Linear(hidden_dim, hidden_dim)\n",
    "        # 对query做运算，decoder的隐藏状态\n",
    "        self.Wq = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, query, keys, values, attn_mask=None):\n",
    "        # query: hidden state，是decoder的隐藏状态，shape = [batch size,hidden_dim]\n",
    "        # keys: EO [batch size, sequence length, hidden_dim]\n",
    "        # values: EO  [batch size, sequence length, hidden_dim]\n",
    "        # attn_mask:[batch size, sequence length],这里是encoder_inputs_mask\n",
    "\n",
    "        # query.shape = [batch size, hidden_dim] -->通过unsqueeze(-2)增加维度 [batch size, 1, hidden_dim]\n",
    "        # keys.shape = [batch size, sequence length, hidden_dim]\n",
    "        # values.shape = [batch size, sequence length, hidden_dim]\n",
    "        score = self.V(F.tanh(self.Wk(keys) + self.Wq(query.unsqueeze(-2))))\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "\n",
    "        #这个mask是encoder_inputs_mask，用来mask掉padding的部分,让padding部分socres为0\n",
    "        if attn_mask is not None:\n",
    "            # 在最后增加一个维度，\n",
    "            # [batch size, sequence length] --> [batch size, sequence length, 1]\n",
    "            attn_mask = (attn_mask.unsqueeze(-1)) * -1e16  # 0还是0，1变为-1e16\n",
    "            # 即填充Pad的区域为-1e16，softmax后，这些位置的权重为0，不会影响输出\n",
    "            score += attn_mask\n",
    "        score = F.softmax(score, dim=-2)  # 对每一个词的score做softmax\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "        #对每一个词的score和对应的value做乘法，然后在seq_len维度上求和，得到context_vector\n",
    "        context_vector = torch.mul(score, values).sum(dim=-2)\n",
    "        # context_vector.shape = [batch size, hidden_dim]\n",
    "        return context_vector, score\n",
    "        # score: 注意力权重，用于画图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6f6fc98dab908ecf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.661714Z",
     "start_time": "2025-01-27T04:36:12.657972Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.390763Z",
     "iopub.status.busy": "2025-01-27T13:00:49.390282Z",
     "iopub.status.idle": "2025-01-27T13:00:49.393054Z",
     "shell.execute_reply": "2025-01-27T13:00:49.392554Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.390741Z"
    }
   },
   "outputs": [],
   "source": [
    "# #tensor矩阵相乘\n",
    "# a = torch.randn(2, 3)\n",
    "# b = torch.randn(2, 3)\n",
    "# c = torch.mul(a, b) #增加维度\n",
    "# print(c.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bc236d1b2f0f089a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.666666Z",
     "start_time": "2025-01-27T04:36:12.663225Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.394050Z",
     "iopub.status.busy": "2025-01-27T13:00:49.393652Z",
     "iopub.status.idle": "2025-01-27T13:00:49.396475Z",
     "shell.execute_reply": "2025-01-27T13:00:49.396024Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.394028Z"
    }
   },
   "outputs": [],
   "source": [
    "# #把上面的BahdanauAttention写一个例子，看看输出的shape\n",
    "# attention = BahdanauAttention(hidden_dim=1024)\n",
    "# query = torch.randn(2, 1024) #Decoder的隐藏状态\n",
    "# keys = torch.randn(2, 50, 1024) #EO\n",
    "# values = torch.randn(2, 50, 1024) #EO\n",
    "# attn_mask = torch.randint(0, 2, (2, 50))\n",
    "# context_vector, scores = attention(query, keys, values, attn_mask)\n",
    "# print(context_vector.shape)\n",
    "# print(scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1517bf2e93e55643",
   "metadata": {},
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "731b9a63e710aaad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.675661Z",
     "start_time": "2025-01-27T04:36:12.667660Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.397510Z",
     "iopub.status.busy": "2025-01-27T13:00:49.397134Z",
     "iopub.status.idle": "2025-01-27T13:00:49.404201Z",
     "shell.execute_reply": "2025-01-27T13:00:49.403661Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.397490Z"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024, num_layers=1):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim + hidden_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "        # 最后分类,词典大小是多少，就输出多少个分类\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        # 定义一个 Dropout 层，随机丢弃 60% 的神经元输出\n",
    "        self.dropout = nn.Dropout(0.6)\n",
    "        # 定义一个 BahdanauAttention 层,得到的context_vector\n",
    "        self.attention = BahdanauAttention(hidden_dim=hidden_dim)\n",
    "\n",
    "    def forward(self, decoder_inputs, hidden, encoder_outputs, attn_mask=None):\n",
    "        # attn_mask是encoder_inputs_mask,用来mask掉padding的部分,让padding部分socres为0\n",
    "\n",
    "        # decoder_inputs.shape = [batch size, 1]\n",
    "        assert len(decoder_inputs.shape) == 2 and decoder_inputs.shape[\n",
    "            -1] == 1, f\"decoder_input.shape = {decoder_inputs.shape} is not valid\"\n",
    "\n",
    "        # hidden.shape = [batch size, hidden_dim]，decoder_hidden,\n",
    "        # 而第一次使用的是encoder的hidden\n",
    "        assert len(hidden.shape) == 2, f\"hidden.shape = {hidden.shape} is not valid\"\n",
    "\n",
    "        # encoder_outputs.shape = [batch size, sequence length, hidden_dim]\n",
    "        assert len(encoder_outputs.shape) == 3, f\"encoder_outputs.shape = {encoder_outputs.shape} is not valid\"\n",
    "\n",
    "        # context_vector.shape = [batch_size, hidden_dim]\n",
    "        context_vector, attention_score = self.attention(query=hidden, keys=encoder_outputs, values=encoder_outputs,\n",
    "                                                         attn_mask=attn_mask)\n",
    "\n",
    "        # decoder_input.shape = [batch size, 1]\n",
    "        embeds = self.embedding(decoder_inputs)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim]\n",
    "        # context_vector.shape = [batch size, hidden_dim] -->unsqueeze(-2)增加维度 [batch size, 1, hidden_dim]\n",
    "        embeds = torch.cat([context_vector.unsqueeze(-2), embeds], dim=-1)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim+hidden_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, 1, hidden_dim]\n",
    "        logits = self.fc(seq_output)\n",
    "        # logits.shape = [batch size, 1, vocab_size]\n",
    "        # attention_score.shape = [batch size, sequence length, 1]\n",
    "        return logits, hidden, attention_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70d7459e2ecbd5de",
   "metadata": {},
   "source": [
    "## Seq2Seq模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "514f7b41823ae373",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:12.685590Z",
     "start_time": "2025-01-27T04:36:12.676662Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.405239Z",
     "iopub.status.busy": "2025-01-27T13:00:49.404942Z",
     "iopub.status.idle": "2025-01-27T13:00:49.415721Z",
     "shell.execute_reply": "2025-01-27T13:00:49.415161Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.405219Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Sequence2Sequence(nn.Module):\n",
    "    def __init__(\n",
    "            self,\n",
    "            src_vocab_size,  # 输入词典大小\n",
    "            trg_vocab_size,  # 输出词典大小\n",
    "            encoder_embedding_dim=256,\n",
    "            #encoder_hidden_dim和decoder_hidden_dim必须相同，是因为BahdanauAttention设计的\n",
    "            encoder_hidden_dim=1024,\n",
    "            encoder_num_layers=1,\n",
    "            decoder_embedding_dim=256,\n",
    "            decoder_hidden_dim=1024,\n",
    "            decoder_num_layers=1,\n",
    "            bos_idx=1,\n",
    "            eos_idx=3,\n",
    "            max_length=512,\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.max_length = max_length\n",
    "        self.encoder = Encoder(src_vocab_size, embedding_dim=encoder_embedding_dim, hidden_dim=encoder_hidden_dim,\n",
    "                               num_layers=encoder_num_layers)\n",
    "        self.decoder = Decoder(trg_vocab_size, embedding_dim=decoder_embedding_dim, hidden_dim=decoder_hidden_dim,\n",
    "                               num_layers=decoder_num_layers)\n",
    "\n",
    "    def forward(self, encoder_inputs, decoder_inputs, attn_mask=None):\n",
    "        \"\"\"\n",
    "        用于训练\n",
    "        (1)根据encoder_inputs计算出eneoder_ouyputs和hidden，用与计算上下文context_vector和scores\n",
    "        (2)然后根据decoder_inputs,context_vector和hidden计算出decoder_outputs和新的hidden，decoder_outputs能算出预测的logits\n",
    "        (3)最后将logits拼接、score拼接，作为模型的输出\n",
    "        注意：训练是一个元素一个元素计算，decoder_inputs其实就是训练样本的正确结果，即logit是由样本提问和回答结果计算得到的，所以训练时，decoder_inputs是正确答案，而非模型预测的结果。\n",
    "        \"\"\"\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_inputs)\n",
    "        # decoding\n",
    "        batch_size, seq_len = decoder_inputs.shape\n",
    "        logits_list = []\n",
    "        scores_list = []\n",
    "        for i in range(seq_len):  # 串行训练\n",
    "            # 每次迭代生成一个时间步的预测，存储在 logits_list 中，并且记录注意力分数（如果有的话）在 scores_list 中，最后将预测的logits和注意力分数拼接并返回。\n",
    "            logits, hidden, scores = self.decoder(\n",
    "                decoder_inputs[:, i:i + 1],  # 取第i个时间步的输入作为decoder的输入\n",
    "                # 取最后一层的hidden，第一个时间步时，hidden是encoder的hidden，第二个时间步时，hidden是decoder的hidden\n",
    "                hidden[-1],\n",
    "                encoder_outputs,\n",
    "                attn_mask=attn_mask,\n",
    "            )\n",
    "            # logits.shape = [batch size, 1,vocab_size]\n",
    "            # scores.shape = [batch size, sequence length, 1]\n",
    "            logits_list.append(logits)  # 记录预测的logits，用于计算损失\n",
    "            scores_list.append(scores)  # 记录注意力分数，用于画图\n",
    "        # [batch size, seq_len, vocab_size], [batch size, seq_len, seq_length]\n",
    "        return torch.cat(logits_list, dim=-2), torch.cat(scores_list, dim=-1)\n",
    "\n",
    "    @torch.no_grad()  # 装饰器，禁止梯度计算\n",
    "    def infer(self, encoder_inputs, attn_mask=None):\n",
    "        \"\"\"\n",
    "        用于预测，\n",
    "        此时，decoder_inputs是开始标记，\n",
    "        模型根据encoder_inputs(即提问)计算出eceoder_ouyputs和hidden，\n",
    "        用与计算上下文context_vector和scores，\n",
    "        然后根据context_vector和hidden生成第一个词，\n",
    "        作为下一个decoder_inputs，用于生成第二个词，作为新的decoder_inputs，以此类推，直到生成结束标记 eos_idx 或达到最大长度 max_length。\n",
    "        \"\"\"\n",
    "        # infer用于预测\n",
    "\n",
    "        # encoder_input.shape = [1, sequence length],这只支持batch_size=1\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_inputs)\n",
    "\n",
    "        # decoding\n",
    "        # 创建一个形状为 (1, 1) 的张量，并将其数据类型转换为 torch.int64\n",
    "        # shape为[1,1]，内容为开始标记\n",
    "        decoder_inputs = torch.Tensor([self.bos_idx]).reshape(1, 1).to(dtype=torch.int64)\n",
    "        decoder_pred = None\n",
    "        pred_list = []  # 预测序列\n",
    "        scores_list = []\n",
    "        # 从开始标记 bos_idx 开始，迭代地生成序列，直到生成结束标记 eos_idx 或达到最大长度 max_length。\n",
    "        for _ in range(self.max_length):\n",
    "            logits, hidden, scores = self.decoder(\n",
    "                decoder_inputs,\n",
    "                hidden[-1],\n",
    "                encoder_outputs,\n",
    "                attn_mask=attn_mask,\n",
    "            )\n",
    "            # logits.shape = [1, 1,vocab_size]\n",
    "            decoder_pred = logits.argmax(dim=-1)\n",
    "            # decoder_pred.shape = [1, 1]\n",
    "            decoder_inputs = decoder_pred\n",
    "            # reshape(-1):从(1,1)变为（1）标量\n",
    "            pred_list.append(decoder_pred.reshape(-1).item())  # 记录预测的词\n",
    "            # scores.shape = [1, sequence length, 1]\n",
    "            scores_list.append(scores)  # 记录注意力分数，用于画图\n",
    "\n",
    "            # 如果生成结束标记，则停止迭代\n",
    "            if decoder_pred == self.eos_idx:\n",
    "                break\n",
    "\n",
    "        # cat(scores_list,dim=-1):[1, sequence length, sequence length]\n",
    "        return pred_list, torch.cat(scores_list, dim=-1)  # 返回预测序列和注意力分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d119467be4a8eb18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.501676Z",
     "start_time": "2025-01-27T04:36:12.686586Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.416857Z",
     "iopub.status.busy": "2025-01-27T13:00:49.416391Z",
     "iopub.status.idle": "2025-01-27T13:00:49.855364Z",
     "shell.execute_reply": "2025-01-27T13:00:49.854724Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.416828Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 50, 12479])\n",
      "torch.Size([2, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx))\n",
    "#做model的前向传播，看看输出的shape\n",
    "encoder_inputs = torch.randint(0, 100, (2, 50))\n",
    "decoder_inputs = torch.randint(0, 100, (2, 50))\n",
    "attn_mask = torch.randint(0, 2, (2, 50))\n",
    "logits, scores = model(encoder_inputs=encoder_inputs, decoder_inputs=decoder_inputs, attn_mask=attn_mask)\n",
    "print(logits.shape)\n",
    "print(scores.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "db1b6485db0bd5b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.509675Z",
     "start_time": "2025-01-27T04:36:13.502671Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.858678Z",
     "iopub.status.busy": "2025-01-27T13:00:49.858354Z",
     "iopub.status.idle": "2025-01-27T13:00:49.862425Z",
     "shell.execute_reply": "2025-01-27T13:00:49.861956Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.858656Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 35,184,576 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "#帮我计算一下model的总参数量\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "\n",
    "print(f\"The model has {count_parameters(model):,} trainable parameters\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d025b8a73c1b5776",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.959275Z",
     "start_time": "2025-01-27T04:36:13.510674Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:49.863297Z",
     "iopub.status.busy": "2025-01-27T13:00:49.863031Z",
     "iopub.status.idle": "2025-01-27T13:00:50.387588Z",
     "shell.execute_reply": "2025-01-27T13:00:50.387073Z",
     "shell.execute_reply.started": "2025-01-27T13:00:49.863276Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 72,970,176 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "#帮我初始化一个4层的GRU的model\n",
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx), encoder_num_layers=4,\n",
    "                          decoder_num_layers=4)\n",
    "print(f\"The model has {count_parameters(model):,} trainable parameters\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39a17b7e22ff2c1d",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a633e80b6643c86",
   "metadata": {},
   "source": [
    "## 损失函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fff18a09726ce0bb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.965493Z",
     "start_time": "2025-01-27T04:36:13.960271Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:50.388514Z",
     "iopub.status.busy": "2025-01-27T13:00:50.388235Z",
     "iopub.status.idle": "2025-01-27T13:00:50.393185Z",
     "shell.execute_reply": "2025-01-27T13:00:50.392657Z",
     "shell.execute_reply.started": "2025-01-27T13:00:50.388493Z"
    }
   },
   "outputs": [],
   "source": [
    "# 用来算一批数据的损失\n",
    "def cross_entropy_with_paddings(logits, labels, padding_mask=None):\n",
    "    # logits.shape = [batch size, sequence length, num of classes]\n",
    "    # labels.shape = [batch size, sequence length]\n",
    "    # padding_mask.shape = [batch size, sequence length]\n",
    "\n",
    "    batch_size, seq_len, num_classes = logits.shape\n",
    "    # F.cross_entropy 是 PyTorch 提供的函数，用于计算交叉熵损失\n",
    "    loss = F.cross_entropy(logits.reshape(batch_size * seq_len, num_classes), labels.reshape(-1),\n",
    "                           reduction='none')  # reduction='none'表示不对batch求平均\n",
    "    # loss.shape = [batch size * sequence length]\n",
    "    if padding_mask is None:  # 如果没有padding_mask，就直接求平均\n",
    "        loss = loss.mean()\n",
    "    else:\n",
    "        # 如果提供了 padding_mask，则将padding填充部分的损失去除后计算有效损失的均值。\n",
    "        # 首先，通过将 padding_mask reshape 成一维张量，并取 1 减去得到填充掩码。\n",
    "        # 这样填充部分的掩码值变为 1，非填充部分变为 0。\n",
    "        # 将损失张量与填充掩码相乘，这样填充部分的损失就会变为 0。\n",
    "        # 然后，计算非填充部分的损失和（sum）以及非填充部分的掩码数量（sum）作为有效损失的均值计算。\n",
    "        # (因为上面我们设计的mask的token是0，所以这里是1-padding_mask)\n",
    "\n",
    "        # 将padding_mask reshape成一维张量，mask部分为0，非mask部分为1\n",
    "        padding_mask = 1 - padding_mask.reshape(-1)\n",
    "        loss = torch.mul(loss, padding_mask).sum() / padding_mask.sum()\n",
    "\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "23808a399cd7ed2c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.972487Z",
     "start_time": "2025-01-27T04:36:13.966487Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:50.394178Z",
     "iopub.status.busy": "2025-01-27T13:00:50.393893Z",
     "iopub.status.idle": "2025-01-27T13:00:50.399711Z",
     "shell.execute_reply": "2025-01-27T13:00:50.399168Z",
     "shell.execute_reply.started": "2025-01-27T13:00:50.394157Z"
    }
   },
   "outputs": [],
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        self.save_dir = save_dir  # 保存路径\n",
    "        self.save_step = save_step  # 保存步数\n",
    "        self.save_best_only = save_best_only  # 是否只保存最好的模型\n",
    "        self.best_metric = -np.inf  # 最好的指标，指标不可能为负数，所以初始化为-1\n",
    "        # 创建保存路径\n",
    "        if not os.path.exists(self.save_dir):  # 如果不存在保存路径，则创建\n",
    "            os.makedirs(self.save_dir)\n",
    "\n",
    "    # 对象被调用时：当你将对象像函数一样调用时，Python 会自动调用 __call__ 方法。\n",
    "    # state_dict() 返回模型参数的字典，包括模型参数和优化器参数\n",
    "    # metric 是指标，可以是验证集的准确率，也可以是其他指标\n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return  # 不是保存步数，则直接返回\n",
    "\n",
    "        if self.save_best_only:\n",
    "            assert metric is not None  # 必须传入metric\n",
    "            if metric >= self.best_metric:  # 如果当前指标大于最好的指标\n",
    "                # save checkpoint\n",
    "                # 保存最好的模型，覆盖之前的模型，不保存step，只保存state_dict，即模型参数，不保存优化器参数\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"01_best.ckpt\"))\n",
    "                self.best_metric = metric  # 更新最好的指标\n",
    "        else:\n",
    "            # 保存模型\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "            # 保存每个step的模型，不覆盖之前的模型，保存step，保存state_dict，即模型参数，不保存优化器参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1ac803ec5a65f989",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.978634Z",
     "start_time": "2025-01-27T04:36:13.973491Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:50.400537Z",
     "iopub.status.busy": "2025-01-27T13:00:50.400356Z",
     "iopub.status.idle": "2025-01-27T13:00:50.404741Z",
     "shell.execute_reply": "2025-01-27T13:00:50.404281Z",
     "shell.execute_reply.started": "2025-01-27T13:00:50.400517Z"
    }
   },
   "outputs": [],
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        self.patience = patience  # 多少个step没有提升就停止训练\n",
    "        self.min_delta = min_delta  # 最小的提升幅度\n",
    "        self.best_metric = -np.inf  # 记录的最好的指标\n",
    "        self.counter = 0  # 计数器，记录连续多少个step没有提升\n",
    "\n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:  # 如果指标提升了\n",
    "            self.best_metric = metric  # 更新最好的指标\n",
    "            self.counter = 0  # 计数器清零\n",
    "        else:\n",
    "            self.counter += 1  # 计数器加一\n",
    "\n",
    "    @property  # 使用@property装饰器，使得 对象.early_stop可以调用，不需要()\n",
    "    def early_stop(self):\n",
    "        # 如果计数器大于等于patience，则返回True，停止训练\n",
    "        return self.counter >= self.patience"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8403c3b46f8ed34",
   "metadata": {},
   "source": [
    "## training & valuating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d782c394b39d9d03",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.985383Z",
     "start_time": "2025-01-27T04:36:13.980577Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:50.405507Z",
     "iopub.status.busy": "2025-01-27T13:00:50.405334Z",
     "iopub.status.idle": "2025-01-27T13:00:50.409652Z",
     "shell.execute_reply": "2025-01-27T13:00:50.409186Z",
     "shell.execute_reply.started": "2025-01-27T13:00:50.405488Z"
    }
   },
   "outputs": [],
   "source": [
    "# 用来算整个test_loader的损失\n",
    "@torch.no_grad()  # 装饰器，禁止梯度计算\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for batch in dataloader:\n",
    "        encoder_inputs = batch[\"encoder_inputs\"]\n",
    "        encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "        decoder_inputs = batch[\"decoder_inputs\"]\n",
    "        decoder_labels = batch[\"decoder_labels\"]\n",
    "        decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "\n",
    "        # 前向计算\n",
    "        logits, scores = model(\n",
    "            encoder_inputs=encoder_inputs,\n",
    "            decoder_inputs=decoder_inputs,\n",
    "            attn_mask=encoder_inputs_mask\n",
    "        )  # model就是seq2seq模型\n",
    "\n",
    "        # 验证集损失\n",
    "        loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "        loss_list.append(loss.cpu().item())\n",
    "\n",
    "    return np.mean(loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f72acf09bc531de1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:36:13.996620Z",
     "start_time": "2025-01-27T04:36:13.988378Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:50.410593Z",
     "iopub.status.busy": "2025-01-27T13:00:50.410383Z",
     "iopub.status.idle": "2025-01-27T13:00:50.418715Z",
     "shell.execute_reply": "2025-01-27T13:00:50.418198Z",
     "shell.execute_reply.started": "2025-01-27T13:00:50.410574Z"
    }
   },
   "outputs": [],
   "source": [
    "def training(model,\n",
    "             train_loader,\n",
    "             val_loader,\n",
    "             epoch,\n",
    "             loss_fct,\n",
    "             optimizer,\n",
    "             save_ckpt_callback=None,\n",
    "             early_stop_callback=None,\n",
    "             eval_step=500,\n",
    "             ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "\n",
    "    global_step = 1  # 全局步数\n",
    "    model.train()  # 训练模式\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            for batch in train_loader:\n",
    "                encoder_inputs = batch[\"encoder_inputs\"]\n",
    "                encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "                decoder_inputs = batch[\"decoder_inputs\"]\n",
    "                decoder_labels = batch[\"decoder_labels\"]\n",
    "                decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "\n",
    "                # 前向传播\n",
    "                logits, scores = model(\n",
    "                    encoder_inputs=encoder_inputs,\n",
    "                    decoder_inputs=decoder_inputs,\n",
    "                    attn_mask=encoder_inputs_mask\n",
    "                )\n",
    "                loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "\n",
    "                # 反向传播\n",
    "                optimizer.zero_grad()  # 梯度清零\n",
    "                loss.backward()  # 反向传播\n",
    "                optimizer.step()  # 优化器更新参数\n",
    "\n",
    "                loss = loss.cpu().item()\n",
    "\n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss,\n",
    "                    \"step\": global_step\n",
    "                })\n",
    "\n",
    "                # 评估\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()  # 评估模式\n",
    "                    # 验证集损失和准确率\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss,\n",
    "                        \"step\": global_step\n",
    "                    })\n",
    "                    model.train()  # 训练模式\n",
    "\n",
    "                    # 2. 保存模型权重 save model checkpoint\n",
    "                    if save_ckpt_callback is not None:\n",
    "                        # model.state_dict() 返回模型参数的字典，包括模型参数和优化器参数\n",
    "                        save_ckpt_callback(global_step, model.state_dict(), -val_loss)\n",
    "                        # 保存最好的模型，覆盖之前的模型，保存step，保存state_dict,通过metric判断是否保存最好的模型\n",
    "\n",
    "                    # 3. 早停 early stopping\n",
    "                    if early_stop_callback is not None:\n",
    "                        # 验证集准确率不再提升，则停止训练\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        # 验证集准确率不再提升，则停止训练\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict  # 早停，返回记录字典 record_dict\n",
    "\n",
    "                # 更新进度条和全局步数\n",
    "                pbar.update(1)  # 更新进度条\n",
    "                global_step += 1  # 全局步数加一\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "\n",
    "    return record_dict  # 训练结束，返回记录字典 record_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e53ee784c3c7e957",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:37:15.031462Z",
     "start_time": "2025-01-27T04:36:13.998439Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:00:50.419750Z",
     "iopub.status.busy": "2025-01-27T13:00:50.419360Z",
     "iopub.status.idle": "2025-01-27T13:11:39.321362Z",
     "shell.execute_reply": "2025-01-27T13:11:39.320861Z",
     "shell.execute_reply.started": "2025-01-27T13:00:50.419730Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 9999/167300 [10:47<2:49:47, 15.44it/s, epoch=5] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stop at epoch 5 / global_step 10000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)\n",
    "test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, collate_fn=collate_fn)\n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx))\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失\n",
    "loss_fct = cross_entropy_with_paddings\n",
    "\n",
    "# 2. 定义优化器\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# 3.save model checkpoint\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(save_dir=\"checkpoints\", save_step=1000, save_best_only=True)\n",
    "\n",
    "# 4. early stopping\n",
    "early_stop_callback = EarlyStopCallback(patience=5, min_delta=0.01)\n",
    "\n",
    "# 训练过程\n",
    "record_dict = training(\n",
    "    model,\n",
    "    train_loader,\n",
    "    test_loader,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=1000\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "43030e8b25455e24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T13:11:39.322702Z",
     "iopub.status.busy": "2025-01-27T13:11:39.322168Z",
     "iopub.status.idle": "2025-01-27T13:11:39.436046Z",
     "shell.execute_reply": "2025-01-27T13:11:39.435464Z",
     "shell.execute_reply.started": "2025-01-27T13:11:39.322681Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAS5hJREFUeJzt3Xd8U+XiBvAnq2kLHdDSFkrZe0iBshFBC8hwXnGAXsStoCBXFO8VlZ8DHBfhKm4FFZkqqOzKkD1aKHvPMtpSoE3pSDPe3x9p0qZJJyc57cnz/Xz6ITnn5OTN29Lz9D3vUAkhBIiIiIgkoJa7AERERKQcDBZEREQkGQYLIiIikgyDBREREUmGwYKIiIgkw2BBREREkmGwICIiIskwWBAREZFktN5+Q6vVikuXLiEoKAgqlcrbb09ERERVIIRAdnY2GjRoALW69HYJrweLS5cuISYmxttvS0RERBJISUlBw4YNS93v9WARFBQEwFaw4OBgyc5rMpmwdu1aDBo0CDqdTrLzkjPWs3ewnr2Hde0drGfv8GQ9GwwGxMTEOK7jpfF6sLDf/ggODpY8WAQGBiI4OJg/tB7EevYO1rP3sK69g/XsHd6o5/K6MbDzJhEREUmGwYKIiIgkw2BBREREkmGwICIiIskwWBAREZFkGCyIiIhIMgwWREREJBkGCyIiIpIMgwURERFJhsGCiIiIJMNgQURERJJhsCAiIiLJKCZYfPLXSfx6Ro1UQ77cRSEiIvJZXl/d1FOWJF3AlRtqXM8xISZM7tIQERH5JsW0WNiXcRUQMpeEiIjIdykmWNgJ5goiIiLZKCZYqOQuABERESknWBAREZH8lBMsCpsseCuEiIhIPooJFrwVQkREJD/FBAs7jgohIiKSj2KChX24KREREclHOcGi8F/2sSAiIpKPYoKFHXMFERGRfBQTLHgnhIiISH7KCRaF/wreCyEiIpKNYoKFHWMFERGRfJQTLHgvhIiISHaKCRaOWMEmCyIiItkoJlgQERGR/BQTLOx3QthgQUREJB/FBAs7jgohIiKSj2KChYrLkBEREclOOcGCt0KIiIhkp5hgQURERPJTTLDgImRERETyU06wcNwKYbIgIiKSi2KCBREREclPQcHC1mTBWyFERETyUUyw4FIhRERE8lNMsCAiIiL5KSZYcFQIERGR/JQTLHgrhIiISHaKCRZ2HG5KREQkH8UECxVHhRAREclOOcGCt0KIiIhkp5hgYccGCyIiIvkoJlhwVAgREZH8FBMsiIiISH7KCRaFnSw4KoSIiEg+igkW7LtJREQkP8UECwc2WBAREclGMcGCw02JiIjkp5hgYccGCyIiIvkoJliwxYKIiEh+igkWdoITWRAREclGMcFCxXEhREREslNMsLBjewUREZF8FBMs2MeCiIhIfooJFkRERCQ/xQUL9t0kIiKST6WChcViwZQpU9C0aVMEBASgefPmeOedd6rFSAzeCSEiIpKftjIHf/DBB/jiiy/www8/oH379khMTMSYMWMQEhKCl156yVNlrBT5Iw4REZHvqlSw2LZtG+655x4MGzYMANCkSRMsWLAAu3bt8kjhKoVNFkRERLKr1K2Q3r17Y926dTh+/DgAYN++fdiyZQuGDBnikcJVRXW4LUNEROSrKtViMXnyZBgMBrRp0wYajQYWiwXvvfceRo0aVeprjEYjjEaj47nBYAAAmEwmmEymKhbbjcI8YTFbpD0vObHXLevYs1jP3sO69g7Ws3d4sp4rek6VqMSf+AsXLsSkSZPw0UcfoX379khOTsaECRMwY8YMjB492u1r3n77bUydOtVl+/z58xEYGFjRty7XjAManLuhwlOtLehYl60WREREUsrNzcXIkSORlZWF4ODgUo+rVLCIiYnB5MmTMXbsWMe2d999F/PmzcPRo0fdvsZdi0VMTAwyMjLKLFhlPfDlDuy7aMBnD3XE4A71JTsvOTOZTEhISMDAgQOh0+nkLo5isZ69h3XtHaxn7/BkPRsMBoSHh5cbLCp1KyQ3NxdqtXO3DI1GA6vVWupr9Ho99Hq9y3adTifph1apVY7y8IfW86T+/pF7rGfvYV17B+vZOzxRzxU9X6WCxV133YX33nsPjRo1Qvv27bF3717MmDEDTzzxRJUKKSUOCiEiIpJfpYLFp59+iilTpuCFF15Aeno6GjRogGeffRZvvvmmp8pXaRwUQkREJJ9KBYugoCDMnDkTM2fO9FBxqk7FVciIiIhkp7y1Qjj3JhERkWwUEyzYXkFERCQ/xQQLO/axICIiko9iggW7WBAREclPMcHCjg0WRERE8lFcsCAiIiL5KC5YcHVTIiIi+SguWBAREZF8FBMsOEEWERGR/BQTLIiIiEh+igkWbK8gIiKSn2KChR37bhIREclHMcGCXSyIiIjkp5hgYccGCyIiIvkoJliwwYKIiEh+igkWdpwgi4iISD6KCRacx4KIiEh+igkWdmyvICIiko9iggXbK4iIiOSnmGBhxy4WRERE8lFOsGCTBRERkeyUEywKscGCiIhIPooJFio2WRAREclOMcHCgZ0siIiIZKOYYMFpLIiIiOSnmGBhx/YKIiIi+SgmWLDBgoiISH6KCRZEREQkP8UFC/bdJCIiko9iggU7bxIREclPMcHCTrD7JhERkWwUEyw4QRYREZH8FBMs7NjHgoiISD7KCRZssCAiIpKdcoJFITZYEBERyUcxwYINFkRERPJTTLCwYx8LIiIi+SgmWHAeCyIiIvkpJlgUYZMFERGRXBQTLDiPBRERkfwUEyzs2MeCiIhIPooJFuxjQUREJD/FBAs7NlgQERHJRzHBgg0WRERE8lNMsLBjHwsiIiL5KCZYqNjJgoiISHaKCRZ2gr0siIiIZKO4YEFERETyYbAgIiIiySguWLDzJhERkXwUEyzYd5OIiEh+igkWdmywICIiko9iggUbLIiIiOSnmGDhwE4WREREslFMsOAEWURERPJTTLCwY3sFERGRfBQTLNheQUREJD/FBAs7drEgIiKSj2KCBbtYEBERyU8xwcKODRZERETyUUywULGXBRERkewUEyzsBDtZEBERyabSweLixYt49NFHERYWhoCAAHTs2BGJiYmeKFvlsMGCiIhIdtrKHHz9+nX06dMHAwYMwKpVq1CvXj2cOHECderU8VT5Ko3tFURERPKpVLD44IMPEBMTgzlz5ji2NW3aVPJCVQUbLIiIiORXqWDxxx9/YPDgwRgxYgT+/vtvREdH44UXXsDTTz9d6muMRiOMRqPjucFgAACYTCaYTKYqFtuVEFYAgMVskfS85Mxet6xjz2I9ew/r2jtYz97hyXqu6DlVohK9Hf39/QEAEydOxIgRI7B7926MHz8eX375JUaPHu32NW+//TamTp3qsn3+/PkIDAys6FuX68cTaiRlqHFfEwv61+cNESIiIinl5uZi5MiRyMrKQnBwcKnHVSpY+Pn5IS4uDtu2bXNse+mll7B7925s377d7WvctVjExMQgIyOjzIJV1oRF+7DiYBpeG9QCT93aTLLzkjOTyYSEhAQMHDgQOp1O7uIoFuvZe1jX3sF69g5P1rPBYEB4eHi5waJSt0Lq16+Pdu3aOW1r27Ytfv3111Jfo9frodfrXbbrdDpJP7RGbRvgotFo+EPrBVJ//8g91rP3sK69g/XsHZ6o54qer1LDTfv06YNjx445bTt+/DgaN25cmdMQERGRQlUqWLz88svYsWMH3n//fZw8eRLz58/H119/jbFjx3qqfJXGCbKIiIjkU6lg0a1bNyxduhQLFixAhw4d8M4772DmzJkYNWqUp8pXYVyEjIiISH6V6mMBAMOHD8fw4cM9URZJsL2CiIhIPopZK4QNFkRERPJTTrAoTBZW9rEgIiKSjWKChX24qdUqc0GIiIh8mIKChe1fs5UtFkRERHJRULCw3QuxsMmCiIhINgoKFraPYmGuICIiko1ygkVh500Lb4UQERHJRjnBovBWiJm3QoiIiGSjmGChtY8KYYMFERGRbBQTLNQcFUJERCQ7xQQLLUeFEBERyU4xwUKtsgcLtlgQERHJRTHBwt5isSjxIqwMF0RERLJQTLC4nmtyPD515YaMJSEiIvJdigkW328753isUnGtUyIiIjkoJlgUZ5/TgoiIiLxLkcHCT6vIj0VERFTtKfIKzM6bRERE8lBksOCQUyIiInkoMlgkHE6TuwhEREQ+STHBonfzuo7H209flbEkREREvksxwWLy4NaOx+uPpstYEiIiIt+lmGDRtn6Q3EUgIiLyeYoJFkRERCQ/BgsiIiKSDIMFERERSYbBgoiIiCTDYEFERESSYbAgIiIiyTBYEBERkWQUGyzyCixyF4GIiMjnKDZY7E25LncRiIiIfI5ig4XgAqdERERep9hgsfEY1wshIiLyNsUGi282n5G7CERERD5HUcGiYx2r3EUgIiLyaYoKFq1D2bGCiIhITooKFkRERCQvBgsiIiKSjKKChUruAhAREfk4RQWLkj0szBZ25iQiIvImRQULS4lk8dWm0/IUhIiIyEcpKliYSjRQfLTmmDwFISIi8lGKChbdwjnclIiISE6KChZBfnKXgIiIyLcpKlgQERGRvBQVLBT1YYiIiGogRV2LVZzIgoiISFaKChbuWKzs0ElEROQtig8WixNT5C4CERGRz1B8sDhy2SB3EYiIiHyG4oOF4J0QIiIir1F8sCAiIiLvUXywEC5LkxEREZGnKC5Y/Gdoa6fnmbkmmUpCRETkexQXLB7v1djp+fL9l2UqCRERke9RXLBwZ8/563IXgYiIyCf4RLC4//NtcheBiIjIJ/hEsACAvAKL3EUgIiJSPJ8JFjeMZrmLQEREpHiKDBbv3ttB7iIQERH5JEUGi3YNguUuAhERkU+6qWAxffp0qFQqTJgwQaLiSMPd6umcKIuIiMjzqhwsdu/eja+++gq33HKLlOWRxC0NQ+UuAhERkU+qUrC4ceMGRo0ahW+++QZ16tSRukw3TaN2bbPo+8EGpGfny1AaIiIi36GtyovGjh2LYcOGIT4+Hu+++26ZxxqNRhiNRsdzg8G2jLnJZILJJN102/ZzlXbOArMVbyw9gNmPxEr2nr6ovHomabCevYd17R2sZ+/wZD1X9JyVDhYLFy7Enj17sHv37godP23aNEydOtVl+9q1axEYGFjZty9XQkJC4SPXj3bobBpWrlwp+Xv6oqJ6Jk9iPXsP69o7WM/e4Yl6zs3NrdBxKiFEhXs1pqSkIC4uDgkJCY6+Ff3790dsbCxmzpzp9jXuWixiYmKQkZGB4GDpRm+YTCYkJCRg4MCB0Ol0aDllrdvjTrwzSLL39EUl65k8g/XsPaxr72A9e4cn69lgMCA8PBxZWVllXr8r1WKRlJSE9PR0dOnSxbHNYrFg06ZN+Oyzz2A0GqHRaJxeo9frodfrXc6l0+k88sNV3nn5Ay0NT33/yBnr2XtY197BevYOT9RzRc9XqWBxxx134MCBA07bxowZgzZt2uC1115zCRVyCq+tR8YNY/kHEhERkWQqFSyCgoLQoYPzrJa1atVCWFiYy3a56bWKnPuLiIioWlPs1TckwH2TzZHLBi5IRkRE5CFVGm5a3MaNGyUohvT+90gs4mdsctk+ZNZmtIkKwuoJ/WQoFRERkbIptsWiRURQqfuOpmZ7sSRERES+Q7HBAgDi20aWuo8dO4mIiKSn6GDx/v2ldyhNPp/pvYIQERH5CEUHi4gg/1L3pWXnw2rliqdERERSUnSwKMt/lh7Ey4uT5S4GERGRovhssACA35MvyV0EIiIiRfHpYEFERETSYrAgIiIiyTBYEBERkWR8PlicTL8hdxGIiIgUQ/HBYtbDsWXuHzd/j3cKQkRE5AMUHyzuiY0uc/+1nAIvlYSIiEj5FB8sypOebcTlrDy5i0FERKQIPhEsmterVeb+TxKOe6kkREREyuYTwaJjdEiZ+wVn9iYiIpKETwSLx3o1LnM/cwUREZE0fCJY6DRlf8xfki7gns+2IDOXHTmJiIhuhk8ECz9t+R9z34UsjP5+lxdKQ0REpFw+ESxaRwbh/s5lDzsFbOGCiIiIqs4ngoVKpcKMh2IrdOw3m057tjBEREQK5hPBojLeW3kEFiu7cxIREVUFg4Ubz89LkrsIRERENRKDhRtrD6fJXQQiIqIaicGiFIM++RvbT12VuxhEREQ1ik8Fi3pB+gofezztBh75ZocHS0NERKQ8PhUsBrWLlLsIREREiuZTwYKIiIg8y6eCRW1/rdxFICIiUjSfChYv3NYC3ZvWxdgBzfFA14Y3dS6LVWDqn4ewYv9liUpHRERU8/nUn/AhgTosfrYXACDfZMEvSReqfK7l+y9hztazmLP1LIbdMkyqIhIREdVoPtViUZy/TlPhY3ecvorH5+zC8bRsrDuShtwCM65kGz1YOiIioprJp1osqiI1Kx8Pf20bdrrx2BUAQHzbSPRsVlfOYhEREVVLPttiUVE9p61z2fbXEc7MSURE5A6DhQQ2HE2XuwhERETVgk8Hi0XP9JTkPGPm7pbkPERERDWdTweLHs3CMLJHoyq99t0VRyQuDRERUc3n08ECANQquUtARESkHD4fLIiIiEg6DBZEREQkGZ8PFqEBfpKcJyvPJMl5iIiIajKfDxbP3tZMkvPM23EOQ2dtRpohX5LzERER1UQ+HyyC/HX474hON32ej9Ycw+HLBnyw+qgEpSIiIqqZfD5YAEDjsEDJzpVvskh2LiIiopqGwQJA18Z10CKittzFICIiqvEYLACoVCr8NfE2Sc618kAq7p29FbkFZknOR0REVJMwWHhAckomFu1OkbsYREREXsdgUUx820jJzpVvskp2LiIiopqCwaKYb/7ZFUffuVOSab5Vxc6xfP8lPPDFNlzOyrv5ExMREVVjDBbFqFQq+Os0CPLXSXrecfP3IvHcdUz943ClX/t/fx7Ga7/sl7Q8REREnsJg4UbrqKCbPkdWngnTVh7B0VSDY9vqQ6n4Pflihc9RYLbi+61nsCgxBSnXcm+6TERERJ7GYOHGrIdjb/ocX2w8ha82ncadMzc7bR+/MLnC5xAQjscmC/tsEBFR9cdg4Ub9kAAE+mk8dn4hnAPDibRsp21uX+Ox0hAREUmHwaIUD8bFeOzcU/8s6mvxzI+JGPjJJixJvOByXPGsUU7uICIiqhYYLErx+tA2+Pqxrh4599xtZ7HuSBqu5RRgw7ErAIBXf92PpHPXcTL9BppMXoGJi5JLvIrJgoiIqj+t3AWorvRaDQa1j/LY+Z/8IdFl2z++2OZ4/Nvei/jX4NaO52yxICKimoAtFtWYsdiCZswVRERUEzBYVGO/JLn2uyAiIqrOGCyqsW82n3Y85q0QIiKqCRgsqjGTpShNiBI3Q4QQyMozOW1LN+TjTEaOV8pGRETkDoNFObZOvh2dG4VixUt9ZS3HwYsGfJJwHAcvZgEAXlqYjE5T12LP+euOY7q/vw4DPt6IjBtGuYpJREQ+jsGiHNGhAVj6Qh+0bxCC4bfUl60cryzZh1nrTmD4p1uQmpWPP/ddAgDMXn8S3285g5PpNxzHnr7CVgsiIpIHh5tWQpdGdbB8/2W5i4Ge09Y5Hq87mo51R9NlLA0REVGRSrVYTJs2Dd26dUNQUBAiIiJw77334tixY54qW7XzWK/GePfeDhjULlLuopSpvOnBiYiIPKVSweLvv//G2LFjsWPHDiQkJMBkMmHQoEHIyfGNpnedRo1HezbGVx6akVMqFqtAgdmKn3eew1l25iQiIi+q1K2Q1atXOz2fO3cuIiIikJSUhH79+klasOpMpVJh1sOxOH81F/9NOC53cVzM2XYWyRcy8eFqW2vS2enDZC4RERH5ipvqY5GVZRuhULdu3VKPMRqNMBqLRikYDAYAgMlkgslkKu1llWY/l5TnLMvQ9hEAUC2DRcLhNCQcTnM8/3LjCdzfuQGe+nEPNGoVbmtVD2P7N6vSub1dz76K9ew9rGvvYD17hyfruaLnVIkq3pC3Wq24++67kZmZiS1btpR63Ntvv42pU6e6bJ8/fz4CAwOr8tbVyvjtNaP/ay2tQI5Z5Xg+saMZjWvLWCAiIqpRcnNzMXLkSGRlZSE4OLjU46ocLJ5//nmsWrUKW7ZsQcOGDUs9zl2LRUxMDDIyMsosWGWZTCYkJCRg4MCB0Ol0kp23PC2nrPXae0npq0c74/bW9Sr9Ornq2dewnr2Hde0drGfv8GQ9GwwGhIeHlxssqvTn9rhx47B8+XJs2rSpzFABAHq9Hnq93mW7TqfzyA+Xp87rCS9pfkOgyogESxfsFS1h9eK0Iv9ZdgiJbwys8utrUj3XZKxn72Fdewfr2Ts8Uc8VPV+lgoUQAi+++CKWLl2KjRs3omnTplUqHAEqWPGYNgH1VFl4TvsnrohgrLd0QYK1K7ZYOyAfrmFMShk3CpCZW4DQQD+Pvg8REfmWSv2JPHbsWMybNw/z589HUFAQUlNTkZqairy8PE+Vr0Z5pHsj/N897St0rBoCU03/xO+W3jCIQNRTGfCQdiO+9fsv9uqfxTe6/2KEZiPCkOWx8n635UyFjvt840n0/WA90gz5HisLEREpQ6WCxRdffIGsrCz0798f9evXd3wtWrTIU+Wr9jpEF91n6twoFP/s1aRCr7NAg+XWXhhvGocuxi8xsuDfmGMejAsiHAGqAgzUJOEj3dfYrX8BS/zexjOaP9FMdUnSsn+6/mSFjvtw9TFcuJ6HmX9VvxEwRERUvVT6Vgg5+350N3y45hhaRNTGA11s/U0GtK6HDceuVPgcZmixzdoB26wdMNX8T7RTnUO8eg8GahLRUX0W3VTH0U19HP/GApyy1keCNQ5rLV2RLFp4tV+GxcrvPxERla1mjJWsxiKC/fHxiE5O274d3Q3p2flQQeW0rkfFqHBYNMFhSxP8z3I/6uMq4jVJGKhOQk/1YTRXX0Zz9Z+y9Msoz5HLBixLvogX+rdASAA7ZxER+SIGCw/QqFWoHxIAwHZ7ZO/5zCqf6zLC8JNlEH6yDEIQcnGbeh8GapIwQJ3s6JfxEDYiT/hhi7Uj1lq7Yr2lM64iRKJPU0QFVZn7h8zaDADIyC7Afx/sVOaxFSWEgEpV9vsSEVH1wWDhYeG1pWtFyEYgllt7Ybm1F7Qwo7v6KAaqkzBQk4SGqgwM1NgeW7UqJImWSLB0xV/WrjgtGkhWhoo4dEmaDqevLNmH/Rcy8eeLfaHXaiQ5JxEReRaDhYd5qluKVP0yUq7lIqZuICxWARUAtbryrQPXcwrwxA+7Jfx0Nr8kXQAAbDx2BYPbR0l+fiIikh6DRTUQ17gOEs9dv4kzuPbLuEOzB4PUieX2y7j1ww04+d4QDPpkEwL1Gnw3uhsig/0B2G5DHLpkKHqXUjLHF3+fcnu7x2IV0JQTVIQQuHA9Dw3rBPCWBxGRAnhvSIGPejDONlKkfYOiYak9mjov2jbrkc6YO6abZO95GWGYZxmIf5peR1fjVxhX8CKWlTFfxtUt3yEr4xIOXjSgx/vrsHh3CgrMVixOTMHwT4vWgbEKgdNXclxaYfIKLC5lOHXlBjq8tQYfrznm2GayWPHRmqPYcfqqY9vUPw/j1g834KGvduB6ToHbz8O4QURUczBYeNig9lFYM6Effn2+t2Nbl8Z1UMuvqM9AdGgA+reOwKrxt0r+/vZ+GRPKmC8jcsMrTvNlfPHbGoz+fhd+2nHO6VyLEy9g8P+2YuPloku9EALJKZku7/vh6qPIM1nw2YaT2HXmGgBg3o5zmL3hFB7+eofjuLnbzgIAdp29hge+3Ob2M7y38gi2ncxw2S6EQMq13EoNg560ZB+e+iGxQq/Zl5KJDUfTK3xuIiJisPCK1lFB8NdpHK0W98Q2wJwx3RFWyw+fPtLZcVzb+tItyuaOvV/GVPNo9DXOwhDjNMwwPQBjvY5Qq4StT4ZuATbo/4V3LzyOx3PmoIvqONSwOp1n2TkNvvz7NADg7+NXcOCic2dNo9mKNYeKlm1/qrD/xZmMnDLLd+pKDt5YdgCHLmUh31TUCnLuai5GfrvT5fhpq47i1g834JvNpytcB0uSLuCvI2k4deVGucfeM3srxszdjXNXyy43EREVYR8LL1o2tg+u5xYgIsjWhyHxjXiXfgXxbSPw1xFv/JWswhHRGEcsjXH/Q/3xyMe/uvbLyP8VD+h/dTtfxn//OokX41tjnZuylgwQ9raBJYkXnLbnGM0ur5234zzm7ThfbulPpGXj6022QPH+yqN4vHdTmK1WZOeb8fpvB/DPXo3Rv3WEczmKtVKYrQKbT1zBsr2X8NqQ1o7viTsXM/PQOKxWuWUiIiIGC6/SadROFzB3nRVnPBSLW952XYo9pm4AUq6VvSaLVq2CuQqzY6pVKke/jHmWgY75MuI1SbjdzXwZO6xtcUbUB7afQ5ssFdqqBC6IeshGoNvzZ+eb8dfhNOQVa4XYdjLDbStERQ38ZJPT81ZvrAIA9G0Rji0nM7D+aDrOTh/mdMxbfxxyPDaZBR77bhcA4Nc9F5yOFUI4WmSA8ufvICKiIgwW1Uywvw6RwXqkGYxO25c82xtphnzcM3ur29ctf7Ev7vvc/b7y9Ptog9Pz8ubLGKDZhwHYB6xZjVEARhVO1ZElAnFB1HN8pRR7POHHXKBY8Hh/1ZFKl3Pi4mTMeDAW20659rew21KiL0ZWngmJZ6+hR7Mw/Lj9XKnHHbiQhY4NbZOKrTuSjg9WH3Xsq8IIXCIin8VgUUNEhfgjKsQfi5/thQe/2u60b8rwdugQLf1Mm4DrfBltVefRVX0cDVVX0C7gOprprsE/5yLCVQaEqHIRojqH9jjn9lyZopYjaOTnRCNZE+wUPnIQUGZZfttzES/d3hIjv6l4S8dj3+3E/guuE3YVDw4A8MxPidj++h0AgAvXc532cRgsEVHFMVhUQ2U1vXdvWhf3xjbAsmTbSqetI4MwulfjYq/z5EJhRf0yAADZRXsCkI9oVQYaqq6goSoDMap0x+OGqisIU2UjVJWDUFUOOuAskL8b95ZYTuS6qI0LItxtq8dFEY4cBGDM3IpPxLXtZIbbUOHO5ax8vP3HIbRz04GWLRZERBXHYFENiXLCwScPxeLBbjFoFl4bUSGldzr0pjz446RoiJOiodv9gU7Bw/YVU+xxXdUN1Cn86oizbs9xTdTGBUM9XNDZg0c4UkSEI3jkwrkuKtuHwz70Nb5tpNP2ArMVJosVOg0HURERlYfBohoKDfBz6WNRnEqlQu/m4S7be7cIw0Y3y7Wv+9dtqBekd9sp1Fty4Y8ToiFOlBI8aiGvRPDIKAwf6WioykAd1Q3ULfy6BWfcnuOqCHIEjpKtHqmiLvLhBzM0KG/Krb+OpDk9H/ntTkSHBmDr5Nur9NmJiMpltQBWs+3LYip8biq2rfBfxzZL4XH2bbbXqwry0eD6bsB8B6CTZ5VpBotq6LORnfHKkn146Y6WuGE0o2MF+098PKITpiw7iNiYUExbVdSHoHm92p4qqmRyEIDjIgbHRYzb/bWR63SrpXjLhz14hKmyEabKRieUPa+FUWhhghZmaGCCFgXQwlS4zfbctt2+rQBamHK0wC8/Axo/QKMr/Lf4Y10p20seU9b+wsdqHSBUnltohqimEAIQ1mIXW7PzV8kLsMVU7AJtKnFBLnYBLvVc5Zzf5QJvke61Et3G1gLoBsBkHAcEyPO7n8GiGmoZGYTfx/Wt9OvCa+vxxaNdAcApWCjBDQTimGiEY6KR2/1BTsHjiksACVUVza2hV5mhR4k5NCrSj+Lgrpv4BJWjA3A3VMDhWoB/CKAPAvTBhf8GAf7BxZ4HF9tmf17sWL9apS/0QsomROGFtMD2ZTUXPbaYK7ndVPhVYLsw2h+73V5sX+E2jdmIftcyoL30UWFYKHnhL+UvcV+mUgNqre2PDbUW0GgLnxf70hTuU2sAtQ5WlQZXMw0IVcl365bBQuFK9hdQqmwE4qhohKOlBA89CuAHM3T2L5XZ6bntsQU6VfHnzse+e1frEr9ki/9bcpu77eW9zvn2lwoCKLhh+7oZKnVhyAgpEUyCnIOIy7agYqEmCND6ez+gWK22ejEbbXXk9K8RMBeU+NdYVK9uX+P6Wo3JiO5pl6FZ/LOtrux/OTpajKR+Dtf9VTmX469ddxf4YoGgmlADqAMAueUcWBEqtZuLbbELrONiqyu84Ba/ABe/KGvLuHDrSpyr9It50TaNdOdTawF15cOBxWTCtpUrMTQwTIKKrhoGC4WLjfHMMNSaxgg/GOFXtKG0VscyWiPf7Tms9J1SKPbXpcmYi3VrVuCOW3tAZ8kD8g2AMRsw2v49fv4SDpw6j4HNayFYlefY7nycwfaXobAC+Vm2r5uh1rlpMSnRiuJXq5wwUFYoKCgKWPZtVtfZWaWmBlAfAG6yemoMtdZ2201d/Padtuh2nLrYYwm3m4UaiXv3Ia57T2j9/MsIAyUvtiW2qTRVuuCS9zBYKFzxORie6tsU325x3/GRyieEuOk5LbadzMClrHw80NVNJ1aVqvAXsxZQ6WDUhQJ1m7vtgDVo6QoA3dE1o47TAnclCgyYcouFjWzAmFX0uGQIcdpWFGJgzAYgbH8Z512zfclF4wdo9IC2tH/1RRezktuc/i16jQVqHDh0GB06doRWU/gr0fF9Vkn8HOUfX9Fz2S+0LiGhRH8fdYmLvUwXZWEyIe20GqK5fJ0KyTsYLHxIl8Z1AAaLKrMKQHOTdwLsQ2Db1Q9GuwaVX3TuwvVcnL5S1F/kRn4Zf9GrVLYWBL9aKPybvFSXMvOgVqncD1+2Wm23Y4qHjfxiYaR4ADFmF13IKnhhd3+czv1rPXArxmoy4VzaSrTvPJQXPCIJMFgoXOdGoY7HVo4yuCm2RcykubBdzMxDuwbBsFoF1BWcgctiFej7gfP06+XNeVIR+SYLek9fDwA4+d4QaEvO16FW225z+AcDiL7p9yMiZWOwUKiNr/TH6YwbTvNd9G8dgWB/LQxl/ZVLpZIyllmsVhxPy8YDX2zD2AEt8Oxtzct9zXdbKr48fEV9knAcW4utm2I0W12DBRFRJfA3iEI1Ca+F29s4jwiprdciacpAfDyik0ylqtmkbPCxCuDN3w/CkG+u8NDg91e6P+5oqgGrD16GxSpgcbO6bb7JguSUTPzz+12YuCjZad+sdSeQeO664/nFzDxczip9Fd0dp69i5+mrFSovEfkmtlj4GJ1GjX90iUa9ID1Gf++9eRmUQIrbDnZ/7rvktCZMXoEFAX4ax3OzxYpjmSqsXJCM+7o0xJ0d3PeREAK4c+Zmx/PGYYFY/6/+0BS7vTJmzm5sLxYGZjwUCwBYeeCyy/kGFS5HP2lwa0QE6TEirmjCstwCMx7+egcA4NU7W2N/ShY+HdnZMdX5heu5eGPZQTzVtxn6tnSdGZaIfANbLHyQSqVC9yZ15S5GjSNli8Wqg6nIKSi6JfXj9rNO+z9cewKfH9FgzeF0PDdvT6nnOZHuPMfFuau5aP7vlVh9MNWxbXuJFoZpq47g282nMXvDyVLP+9GaY5j0y/7CfiVAxg0jbhiLyvvh6mNYfSgVK/YXhZO+H2zAxmNX8Oh3lVujhYiUhcHCR5U14kzD5Tzdkrrvq9FkdTz+eO0xmC1Fz+dsc7/0fEU9Ny8JvyZdwJVs1zVnvvr7NN5dcQRmS/kfyGIV+HD1UcS9+xemu7kVk1NghtFsgbXELRh3t2SIyDfwVoiP0pWRLI78351o9cYqL5amZrjZWyHrjzovblZ85KTJIjBt1VHkmSzIcBMGquJfS/aVuf9YWnaZ+wHg553n8fnGUwCA3/ZedNn/v3Un8N6KI6itd/5VcuSyAR0quMYNESkLWyx8lFqtwtqX+2H5i85rkjx7WzP4adXYOvl2PHtbM8f2puG1nI47OnWgV8pZneSbrHjh5yT8tOMcTl8pe5pti1Vg1YHLSM3Kd2x7Ym6i0zHZJUbnfLflDObvPI+1h50DiJze+uNQmfvTDEbkFliQXiIMvfn7QU8Wi4iqMQYLH9YqMsjpr8rtr9+O14e0BQBEhwZgeMcGjn1tooKcXlva7ZIT7w0p932nDG+H/q3rlXvcne2j4FeNhj7+tP0cVh5IxZRlB3H7f/9GckomAFtHy5Rrufg9+aLjlsD8nefw/M970HPaOlzKzHO5VQDYRmBUlKhhc5AcuOgr82MTUUm8FUJY8HRPZOebUD8kwGl7x4Yh+PSRzoipG4iEw0WdAV+9xf08GPWC9NCW0z9DpQKe7NsUT/Ztiux8E778+xRmbzjl9tj4dpF4fWgb3PbRRrf7f3muF1YcuIw5W8+W+Z52w26p79TZEABejm+FT/46XqHXL9p93un5vbO34tDUweg1bZ1jbpB8kwUhAX6Y8nvRX/r2yaduRtPXV970ObzpZqc+J6Kaq/r8OUiy6dU8DIPaR7ndd1enBoiNCcXYAS0wpk8T/PxkHKIL74oce/dOp2O3Tb4dKpXKabbPW1uG4+z0osW7NMUuOEH+Okwa3AavDGqFWQ/HYkiHKLStH4xWkbXRqWEI7o1tgMZhtfB47yZO77Nq/K34fFQXxDWpizsK5+rQqFUY2C4Sw29xHpa58qVbi54IYO6Ybk777+scjf1vDyqzfuwuFbutYdf+rTVOE4699usBPDcvqULnUzJfjRULd53HY9/tRHa+jy/3TT6NLRZUIYF+Wrx1V3uYTCasPGzbptdq8MtzvfDQ1zvw+pA2jvkMfnyiOzq+vRYAUMuv/B+xcbe3BADcExvtdqGvt+9uj+x8M37dcwFRwf5oWz8Ybevb1tno2zIcvzzXC03CayG8th4AsHz/Csdr2zUIxj2xDfB78iU83a8ZYmNCceT/7kTbN1cDsHXIDPbn+hBSU1fTFgshBF5Zsh/RdQIwcWCrmz7fuPl7kHHDiPlP9YRarcLk3w4AAL7dfAYvS3B+opqILRZ0U+Ka1MWxd+7EU7cWdfQM8tchOtR2W2VIR+eWkPKuN6U1oU+9pz3eHN4Ov73gupJnXJO6jlABADMe7IQO0cHYOvl2AMDMh2Kx/+1BiI0JrVAZ7D74R8eKHUguquuI5YMXDfh1zwX8b92JSr/2tz0XMHTWZpzJsC0CJ4TA8v2XseP0NZf5RLLzzTielo0BH2/E78kXIYTAwYtZKDBb3Z2aSFEYLOimuVtbYuVLt2Lxs71wd6cGTtvtgaOyauu1eKJvUzSowOvv79IQy1+81fFeKpXKqVWivGDxwxPd8e+hbfBgXAzuLOUWEZUtp8CCsxk55R8oMaPZ4rgNkW+y4Kcd55ByLddpv932U2VPTS6EcHS6zco1YeLifTh82YABH29EmiEffYr1nSm5wJ9KBby8KBlnMnIwfmEynp+3B8M/3YIXft5T4zrietq1nAI88vUOLHMznJlqJgYL8oiQQB26N63raIH49fne6N+6Hr4d3a2cV3qX/Xf8+Dtst2P6tAjDba3q4Zl+zaFSqfD5qC4ylq5m6//xRqQbXPuleFKvaevR8e21SM/OR5spqzFl2UEM/ORvt8c+8s0OxwRihgLgpYX78PDX23GwcETLkz8kYtDMTTBZrC4jeMbN3+PU5ybfZHFaY+W7LWdw6JLB8Xz1IVvn57+OpKHp6yvRZPIKNJm8AkcuFx3jqz5eewzbT1/FhBLr2FDNxT4W5BVdG9fB3DHd5S4GAMBPo0bfFuG4YTSjUd1AALZgcWeHKLSKdB5WW027CtQYhy4bYMg3oXm92pKOFLFf6Eu2gF3LKQAAvFBsGvR8kxX/XnoAA1pHoE6gc3+aU1duYMW+i/gpSQvANn/I8E+34Oz0YVh/NB2AbQVY+yRhdrvPXnd6ft/n26r0OYbM2ozX7myD5/uXv7qtUmXlFXV0ddfHimoetliQz1GpVPjpye5Y+kJvqAs7A6jVKrStH+wyP0d1/iU3/6keeOee9nIXo0zTVh5B/IxN+HjtMUnOl2M041pOAfpMX48+09c73doofouhZJ+H+TvP4+kfEx1zj9h9tOYYftqZ4vI+M4sNQS4ZKqT2weqKrW4rB0/ctvnrSDp2lLJC7mu/7pf8/cj7GCzIJ6lUqmodGioiwE+Dx3o1wdqX+8ldlFIdT7Nd4Eubq6SiLFaBp39MRPu31qDLOwmO7ddyCrDtZAYKzFanxc+K/xVc3Lsrjjg9P3TJ/UReM/+qfOdOpVm0+zzi3v3LcWtICteMwPPzkx2r5Ja0OPGCZO9F8mGwIKqh7MGoVWQQ7oltUM7R1cPpKzeQU7hK6s87z+HlRckoMFtxMv0Gxs3fg8W7i1oPUrPyHR1Av950Gglupjp/ft4ejPx2J1pPWYWtJ8vujOlOvomjNErz2q8HcDWnAC/fZN8HIQTGL9yL/1t+BIaCqp1j47F0NJm8At9tOXNTZalJhBB44eckvFLOmj/VEftYENVQxdtbZj3cGb8nX5KtLBWx/0Im7v5sKyKD9dj573j8Z6ltPZGlxUYDLN9/Gf3b1EOAToOe09YBANrWDy61k6P91gYHWnhOyREvAPB78kVEBPmjU0wIAkvMVbPn/HW8+st+vDm8Hfq1qofTGTmOn82XO1T8fc0WK7QaNYQQeHzObgDAO8sP48m+Tav+YWqQC9fzsPKArdPve/d1gF6rkblEFcdgQVSD1NZrcaPwL/76If5O+6KC/ZFahVEY5U1r3iqytuOWxs24+7OtAGwLl00u41569/fWOT3nyAl55RYU9WPJMZrx5u+H8OueolsWR9+5E/66ooveo9/uRG6BBf/8fheOvnOno0MtAJzJLorD+SaL7XVuQuHe89dx3+fbMGlwa6dzA4DVKhx9o5Qsp6BoRt+aFpwZLIhkEKDTIM9kcdr2v0c646UFe8t83R/j+iDfZMUNoxkRwc7B4q9/3YZLmXlQq4D4GZuc9vVpEVbqrQKLtezbAQue7onxC5PRIToEp6/ckGT11YW7XTtMUvV0udiw2vdXHnEKFQBw/louWkUGYc2hVPx1OM0piIz4crvTgnTrLxXdfW8zZTXCa/sh44bz/ZEmk4tmzv1ojWun3z/2XcK9naOr/oFqgK0nMzDq253lH1hNMVgQedHmVwfg8GUDBrWLdFpY7LORnTGsY30s23vRMczxmTYW7DOGY+eZoqGNdQL9UKeWn9tz19ZrXYbLAsDXj3VFfNtIZBvN+CThOOZuOwsAmPVwLGJjQjF/53mX1wC2YbkqFRAa6Id5T/UAAKfg80y/ZnimXzP0/2ijoxWFvCvHaEaB2Vrqz4Q7pf3Fb7EKJBxOw8Ld5/HOPc73LJpMXoGpd7fHz25+VgZ9ssllm13JVW4NJuf3LRkqKmLR7pRqEyyEEDh3NReNwwLL7Aze94P1uHA9D0ffuRNmq0BtfemX3kuZeZUOFYt3p0CrUeH+Lg0r9TpPYbAgKseKl/oiM9eExYkppfZjCA3UYXSvJni+f3O0mWJbh+ThbjGOv8xbRtTGlOHtEFM3EDGFc2d89VhXPPuTbcGy4bfYOl9+/3g3XMrMw7mMbKQf2o5JQ7shM9+Kbu/9BaDi82o80LUhfkm6gFcGtXIsMBcSoMPkIW0QEazHoHZRaBFRGwDQPjrE5fVvDm+HkT0aAYDTENz7ukTjj32X0DKiNv49tC0AYO+bA3Hxeh7GL0rGvhLDOalqhBDYdeYaWkYGoW4ZoaH9W2sAAL2aheG7x+Pw3eYz2HwiA5+O7IzIYi1aBWYr/LRqzNl6BlP/PIzPRnZG7+bhqBOow5HL2Xjwq+1O4fDWDze4vNdbfxxy2SaH0kb8yOGz9Sfx34TjeKZfM8f/h5LMFisuXLfNu2L/3fDHuD64pWGo4xghBPZfyMKe89cx9c/DLueYs/Us7o5tgAYh/lCpVNhx+iqy880Y2C4SZzNy8GrhrcVdZ66hX4swiT9l5amEl+eXNRgMCAkJQVZWFoKDgyU7r8lkwsqVKzF06FDodFxUylN8vZ6LN9N+NzoOExYlY8rwdngwLsaxfcuJDOy7kInnbmuO5v+2tUpsfnWAI1AUN3vDSYTV8sPD3Rs5bS9ez1qtFs/P2wONWoXZFZwJ1Gyx4kT6DbSJCip3WK3VKvDuiiM4mmrAp490htkqnC5KJR1LzUajuoEI8HPtTPbE3N2OFhequOIrAAPA6oOpeG5eEuoE6rD3zdJX3y3+8zi4fSTWHLLdpooODcATfZuiWXgtBPlr8cCX29GtSR2Xib00ahUs1pp1A79TwxD8Pq6vrGWwjdjYg1UHUx3bin8Ppyw7iIOXsrDg6Z44kXYDd322xeUcH/yjIwa0iUCAToNlyZcwZdnBCr33mWlDnVo73ZnezYx/3C397+iKXr/ZYkFUCcU7T97RNhIH3h7sckzfluHo2zIcALD25X64nlPgNlQAwNgBLcp9T5VKhS8f61qpcmo1ascKsOVRq1V48652FT536yjX2y2Oc9XwuUE8aeyA5mXO53EmIwebT1zBQ91i8Nw8W0vW9VwT/tx3CbExoQj00+DH7edwb+doBPtrEVZs4T0AjlAB2GYmfWe581++JUMFgBoXKgDAXMkyz95wEosTU/DKoNZo1yAYzevVvukypGcbnUIFAGw6fgVzt51FmiHfMZ177+nrnTqvFvfarweq9N4TF5c//HTxaTX+UaWzS4PBgqgSdv8nHrM3nMQ/ezeu0PHu+jwoWXq2d9cG8ZY9UwZi15lrjgt+VTxza+nBYt6Oc3ij8C/WN393vuXwYokOvbMKV2Z9vHeTKpelJutUuEpxaa5kG3Hqyg30KFyryN4B1F6Pc8d0Q6vIIJcFDWdvOIn5O8/jm3/GoV2DYOw6cw3nr+XiH12ioVKpcO5qDnaevobruQUYdkt9l/f95/e7XLaVFipuxtIKLNa256q8U1QxWBBVQoCfBq8Mbi13Maqt/Rekm6WxOqlbyw93dojCxlf6IzxIj1p+GlzOyseWExmO+9tl8dOoERKow8B2kW4n+nqjgs3gxdk74fqa7Hwzks5dx+LdKViUmIK37mqHMX2K5rbo88F6FJiteKBrQ7zlpiXOPifG3DHd0L91hGO7PYAM/d9mvHtvB8f35Py1XOQazfi22ORc01ZV32nYqwMGCyLyCfZ5PoZ0iMKMB2Ox4/RVjJm7u1LnaBJey/G4QWgA7u0c7RIsWkcG4VhaNto3CEa3JnUxd9tZvDHc1rHvq0e74rHvd1ZpllCy+XPfJfy5r6gT9dQ/DyO+bSRu/XADejcPQ4HZNnz6l6QLLovOFff4nN14sm9THLyYhSs3jE77ige9/62rmdO7X8spQGSoPP3gGCyISDK1/DTIKbCUf6AM7usSjX8NbAWtxtZMPKBNBA5NHYwvNpyAMfUEvjla+ZkN/bRqvHVXO0dP/kmDW2NE14ZYsCsFD3ePQUSQHk/2beroY6NWq/DzUz0B2G4blZwMjKrGPopl2ynnwPbN5rKnAFfyFOFeHpfhhGuFEJFkRt/Eff+X41s5PY+pW3QPfNbDsY7Ht7eJwC/P9cLjvZtgeOG97o2v9MemSQNcznl/l6L5DsJr6x2hwq6WXovxd7RAhzoCm17ph6hgfzzZtym2v357mXMNFPdI90bo0yIMbwxri7EDWiAi2B/j41siMtg2NLC0jrt+Gv76Jc/RqOX7+WKLBRFJZlTPxqUuM/7RA7dg4e4UJJ0rGp1wS8MQ7L+QheUv9kWH6BBsO5WBnWeuAbDN+Ll4dwoe7dUYEUH+uCc2Gueu5iA6NABajRpxTeoCAP77oMWxjkLSG/FIuZ6HSUv24dGejTG6dxMMbBuJdUfT8WjPRq6FKqZ+iD+2v367Y3ju9493wzM/JWLq3WUvTe+v0zhaISqjpq+u601/TbwN8TP+lrsYNYpGxmnPGSyISDLRoQE4+s6d+PdvB/Bbid7rI+JiMCIuBleyjThy2YBbW4ZDpVJBCOG4yP73wU54ft4ePNm3KRrWCcTEQc4dZRuH1UJJxRdnCqutR1htPRIm3ubYNqRjfQzp6NqL353iF/vuTeti75SBHgsAwf7V/9fvhPiWCA3Q4dP1J3G1nBEOxYdi/zGuD7afuooH42Ics4KOnZeEFYVDNKcMb4cgvRaz1p3AJw/F4v+WH8LBi65rwjSrVwufj+rimMyNKq62Xr5Fy6r/TzYR1Sj+Og2m3tMeraOCMLRjfRy+bHCacKtekB71guo5nhe/cDesE4g/X5R38qPiPNmqUF1aLOytRu5MKLw99XifpriclYfxC5IxqH0kDHkmxDWpi+g6AdBr1agXpIdeq8GF67kQAoipG+g0syQATL27LZB1Ef8acRuaRdhme32wm21iueUv3ooCsxU6jQrnr+WiUeHto+pSR9VZXOM6SDznOkeJnHXHYEFEkgvy1+HZ25oDQKl9DMjzZj4UiwmLkkvdP+vhWNwTG42VBy7jhZ/3lHmu+iEBWPxcrzKPaVin9O91SIAOgxoKxJRyjJ/W1ifAXasUAHRtXMfpNlpNM3tkF9SppcPaQ2mOocJLnuuFLzaecpmt9sm+TaHTqBEaqMP0EkNbH+3ZCCO6xmDVwVSMu70Fauu1WJKYgkm/FI1OqquXd+IzBgsiIpl8NrIzxs0vmgBrRNeGOJOR4/IXqF6rxoqXbnX0M7ilYQheHtgKAwrnYbBaBY6mZmP4p5vx1K3NkFtgRqCfFvd2jsa9naORmpWPR7/biZPpNwAAD8Y1xJ7zmRhcuI7M0I718fGITpi77Qxmj+yCN38/hO5N63qjCipswdM90eqNVXIXo0KC9FpkG83Qa9XYNvl21NJrHcu/92gahvu7RKNd/WBoNWp0e7wuhBCYu+0slu69iOx8MybEt0SQv22o6KqDqcjINmLZ2D5ITsnEgNb1oNWonSYKGxEX4wgWg9pFoJe/+zWNvIXBgohIJsNvaYBB7aJwLacAGTeM6FC4IFzKtVyMW7AXeq0au85cw6yHY9EiojY2vzoAdWv5oVaJEStqtQrtGgTj1PtD3TaBR4X446+Jt+FiZh50ahUigv2d+rYAtoXrHuhqWx3zhye6e/BTV42fVo1+reph0/ErAIAZD3bC2au5Xptn4tGejfDOPR0c63QseqYnOsWEYtS3O9G8Xi0M7VgfHaJDEBKgw+krOfh840lMiG/lMvW6Rq1yuU2kUqkwpk9Tp4m+7JY+3xtWIaDVqDGwXWSp5Zv3ZA9czMzF/bH1sXIlgwURkc/y06oRFeKPqJCifigxdQPx+9g+AGxLo9uDRHm3lcq7rx5dbBrrmth/4ccnumPv+euIrhOAiCBbfUUF+6N5vVro0cy2qqfRbIFWrcYzPyZiXbFbDL2bh2H+0z1RYLbivs+3YkTXhjh7NRdDO9Z3tM6kGfIxZs5ujOzRCA/GxeDrTafw8drjAIB37+0IAPh7Un+cychxvN+vz/d2KWfrqCDMerizJJ9ZrVZBjfK/V/b1iUwm+Vd/ZbAgIqrGSrZO+LrOjeo4PR/Zw3kYsX2U0Gcju2D32Wvo2SwMhnwT6gTaRqf4Fd5Wcicy2B8rxxfte6Zfc4QG+uHWwos2YOsDUlo/ELLhTywRESlOgJ8G/VrZRh+Fl7gdUVF+WjUe7VmxBQepCKd+IyIiIslUKVjMnj0bTZo0gb+/P3r06IFdu1yXiyUiIiLfU+lgsWjRIkycOBFvvfUW9uzZg06dOmHw4MFIT08v/8VERESkaJUOFjNmzMDTTz+NMWPGoF27dvjyyy8RGBiI77//3hPlIyIiohqkUp03CwoKkJSUhNdff92xTa1WIz4+Htu3b3f7GqPRCKOxaK17g8E2H7zJZJJ0WIz9XNVhqI2SsZ69g/XsPaxr72A9e4cn67mi56xUsMjIyIDFYkFkpPMkHZGRkTh69Kjb10ybNg1Tp0512b527VoEBko/1W9CQoLk5yRXrGfvYD17D+vaO1jP3uGJes7Nza3QcR4fbvr6669j4sSJjucGgwExMTEYNGgQgoODJXsfk8mEhIQEDBw4EDqdTrLzkjPWs3ewnr2Hde0drGfv8GQ92+84lKdSwSI8PBwajQZpaWlO29PS0hAVFeX2NXq9Hnq96xhinU7nkR8uT52XnLGevYP17D2sa+9gPXuHJ+q5ouerVOdNPz8/dO3aFevWrXNss1qtWLduHXr1KnvVOyIiIlK+St8KmThxIkaPHo24uDh0794dM2fORE5ODsaMGeOJ8hEREVENUulg8dBDD+HKlSt48803kZqaitjYWKxevdqlQycRERH5nip13hw3bhzGjRsndVmIiIiohuNaIURERCQZr69uKoQAUPFhKxVlMpmQm5sLg8HAHscexHr2Dtaz97CuvYP17B2erGf7ddt+HS+N14NFdnY2ACAmJsbbb01EREQ3KTs7GyEhIaXuV4nyoofErFYrLl26hKCgIKhUKsnOa594KyUlRdKJt8gZ69k7WM/ew7r2Dtazd3iynoUQyM7ORoMGDaBWl96TwustFmq1Gg0bNvTY+YODg/lD6wWsZ+9gPXsP69o7WM/e4al6Lqulwo6dN4mIiEgyDBZEREQkGcUEC71ej7feesvtuiQkHdazd7CevYd17R2sZ++oDvXs9c6bREREpFyKabEgIiIi+TFYEBERkWQYLIiIiEgyDBZEREQkGcUEi9mzZ6NJkybw9/dHjx49sGvXLrmLVG1NmzYN3bp1Q1BQECIiInDvvffi2LFjTsfk5+dj7NixCAsLQ+3atfGPf/wDaWlpTsecP38ew4YNQ2BgICIiIjBp0iSYzWanYzZu3IguXbpAr9ejRYsWmDt3rqc/XrU1ffp0qFQqTJgwwbGN9SyNixcv4tFHH0VYWBgCAgLQsWNHJCYmOvYLIfDmm2+ifv36CAgIQHx8PE6cOOF0jmvXrmHUqFEIDg5GaGgonnzySdy4ccPpmP379+PWW2+Fv78/YmJi8OGHH3rl81UHFosFU6ZMQdOmTREQEIDmzZvjnXfecVo3gvVcNZs2bcJdd92FBg0aQKVSYdmyZU77vVmvS5YsQZs2beDv74+OHTti5cqVlf9AQgEWLlwo/Pz8xPfffy8OHToknn76aREaGirS0tLkLlq1NHjwYDFnzhxx8OBBkZycLIYOHSoaNWokbty44TjmueeeEzExMWLdunUiMTFR9OzZU/Tu3dux32w2iw4dOoj4+Hixd+9esXLlShEeHi5ef/11xzGnT58WgYGBYuLEieLw4cPi008/FRqNRqxevdqrn7c62LVrl2jSpIm45ZZbxPjx4x3bWc8379q1a6Jx48bi8ccfFzt37hSnT58Wa9asESdPnnQcM336dBESEiKWLVsm9u3bJ+6++27RtGlTkZeX5zjmzjvvFJ06dRI7duwQmzdvFi1atBCPPPKIY39WVpaIjIwUo0aNEgcPHhQLFiwQAQEB4quvvvLq55XLe++9J8LCwsTy5cvFmTNnxJIlS0Tt2rXFrFmzHMewnqtm5cqV4j//+Y/47bffBACxdOlSp/3eqtetW7cKjUYjPvzwQ3H48GHxxhtvCJ1OJw4cOFCpz6OIYNG9e3cxduxYx3OLxSIaNGggpk2bJmOpao709HQBQPz9999CCCEyMzOFTqcTS5YscRxz5MgRAUBs375dCGH7j6BWq0VqaqrjmC+++EIEBwcLo9EohBDi1VdfFe3bt3d6r4ceekgMHjzY0x+pWsnOzhYtW7YUCQkJ4rbbbnMEC9azNF577TXRt2/fUvdbrVYRFRUlPvroI8e2zMxModfrxYIFC4QQQhw+fFgAELt373Ycs2rVKqFSqcTFixeFEEJ8/vnnok6dOo56t79369atpf5I1dKwYcPEE0884bTt/vvvF6NGjRJCsJ6lUjJYeLNeH3zwQTFs2DCn8vTo0UM8++yzlfoMNf5WSEFBAZKSkhAfH+/YplarER8fj+3bt8tYspojKysLAFC3bl0AQFJSEkwmk1OdtmnTBo0aNXLU6fbt29GxY0dERkY6jhk8eDAMBgMOHTrkOKb4OezH+Nr3ZezYsRg2bJhLXbCepfHHH38gLi4OI0aMQEREBDp37oxvvvnGsf/MmTNITU11qqOQkBD06NHDqZ5DQ0MRFxfnOCY+Ph5qtRo7d+50HNOvXz/4+fk5jhk8eDCOHTuG69eve/pjyq53795Yt24djh8/DgDYt28ftmzZgiFDhgBgPXuKN+tVqt8lNT5YZGRkwGKxOP3iBYDIyEikpqbKVKqaw2q1YsKECejTpw86dOgAAEhNTYWfnx9CQ0Odji1ep6mpqW7r3L6vrGMMBgPy8vI88XGqnYULF2LPnj2YNm2ayz7WszROnz6NL774Ai1btsSaNWvw/PPP46WXXsIPP/wAoKieyvodkZqaioiICKf9Wq0WdevWrdT3QskmT56Mhx9+GG3atIFOp0Pnzp0xYcIEjBo1CgDr2VO8Wa+lHVPZevf66qZUvYwdOxYHDx7Eli1b5C6K4qSkpGD8+PFISEiAv7+/3MVRLKvViri4OLz//vsAgM6dO+PgwYP48ssvMXr0aJlLpxyLFy/Gzz//jPnz56N9+/ZITk7GhAkT0KBBA9YzOanxLRbh4eHQaDQuPenT0tIQFRUlU6lqhnHjxmH58uXYsGGD01L2UVFRKCgoQGZmptPxxes0KirKbZ3b95V1THBwMAICAqT+ONVOUlIS0tPT0aVLF2i1Wmi1Wvz999/43//+B61Wi8jISNazBOrXr4927do5bWvbti3Onz8PoKieyvodERUVhfT0dKf9ZrMZ165dq9T3QskmTZrkaLXo2LEjHnvsMbz88suO1jjWs2d4s15LO6ay9V7jg4Wfnx+6du2KdevWObZZrVasW7cOvXr1krFk1ZcQAuPGjcPSpUuxfv16NG3a1Gl/165dodPpnOr02LFjOH/+vKNOe/XqhQMHDjj9MCckJCA4ONjxS75Xr15O57Af4yvflzvuuAMHDhxAcnKy4ysuLg6jRo1yPGY937w+ffq4DJc+fvw4GjduDABo2rQpoqKinOrIYDBg586dTvWcmZmJpKQkxzHr16+H1WpFjx49HMds2rQJJpPJcUxCQgJat26NOnXqeOzzVRe5ublQq50vGRqNBlarFQDr2VO8Wa+S/S6pVFfPamrhwoVCr9eLuXPnisOHD4tnnnlGhIaGOvWkpyLPP/+8CAkJERs3bhSXL192fOXm5jqOee6550SjRo3E+vXrRWJioujVq5fo1auXY799GOSgQYNEcnKyWL16tahXr57bYZCTJk0SR44cEbNnz/apYZDuFB8VIgTrWQq7du0SWq1WvPfee+LEiRPi559/FoGBgWLevHmOY6ZPny5CQ0PF77//Lvbv3y/uuecet8P1OnfuLHbu3Cm2bNkiWrZs6TRcLzMzU0RGRorHHntMHDx4UCxcuFAEBgYqehhkcaNHjxbR0dGO4aa//fabCA8PF6+++qrjGNZz1WRnZ4u9e/eKvXv3CgBixowZYu/eveLcuXNCCO/V69atW4VWqxUff/yxOHLkiHjrrbd8d7ipEEJ8+umnolGjRsLPz090795d7NixQ+4iVVsA3H7NmTPHcUxeXp544YUXRJ06dURgYKC47777xOXLl53Oc/bsWTFkyBAREBAgwsPDxb/+9S9hMpmcjtmwYYOIjY0Vfn5+olmzZk7v4YtKBgvWszT+/PNP0aFDB6HX60WbNm3E119/7bTfarWKKVOmiMjISKHX68Udd9whjh075nTM1atXxSOPPCJq164tgoODxZgxY0R2drbTMfv27RN9+/YVer1eREdHi+nTp3v8s1UXBoNBjB8/XjRq1Ej4+/uLZs2aif/85z9OwxdZz1WzYcMGt7+TR48eLYTwbr0uXrxYtGrVSvj5+Yn27duLFStWVPrzcNl0IiIikkyN72NBRERE1QeDBREREUmGwYKIiIgkw2BBREREkmGwICIiIskwWBAREZFkGCyIiIhIMgwWREREJBkGCyIiIpIMgwURERFJhsGCiIiIJMNgQURERJL5fymswgTKFiH8AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画图\n",
    "plt.plot([i[\"step\"] for i in record_dict[\"train\"]], [i[\"loss\"] for i in record_dict[\"train\"]], label=\"train\")\n",
    "plt.plot([i[\"step\"] for i in record_dict[\"val\"]], [i[\"loss\"] for i in record_dict[\"val\"]], label=\"val\")\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caf2b00ee2704d39",
   "metadata": {},
   "source": [
    "# 推理\n",
    "\n",
    "- 接下来进行翻译推理，并作出注意力的热度图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "913a1278f74da64b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:37:15.034459Z",
     "start_time": "2025-01-27T04:37:15.034459Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:11:39.437097Z",
     "iopub.status.busy": "2025-01-27T13:11:39.436828Z",
     "iopub.status.idle": "2025-01-27T13:11:39.761411Z",
     "shell.execute_reply": "2025-01-27T13:11:39.760806Z",
     "shell.execute_reply.started": "2025-01-27T13:11:39.437074Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx))\n",
    "model.load_state_dict(torch.load(\"checkpoints/01_best.ckpt\",weights_only=True, map_location=device))\n",
    "\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.model = model\n",
    "        self.model.eval()  # 切换到验证模式\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "\n",
    "    def draw_attention_map(self, scores, src_words_list, trg_words_list):\n",
    "        # 绘制注意力热力图\n",
    "        # scores.shape = [source sequence length, target sequence length]\n",
    "\n",
    "        # 注意力矩阵,显示注意力分数值\n",
    "        plt.matshow(scores.T, cmap='viridis')\n",
    "        # 获取当前的轴\n",
    "        ax = plt.gca()\n",
    "\n",
    "        # 设置热图中每个单元格的分数的文本\n",
    "        for i in range(scores.shape[0]):  # 输入\n",
    "            for j in range(scores.shape[1]):  # 输出\n",
    "                # 格式化数字显示\n",
    "                ax.text(j, i, f\"{scores[i, j]:.2f}\", va='center', ha='center', color='k')\n",
    "\n",
    "        plt.xticks(range(scores.shape[0]), src_words_list)\n",
    "        plt.yticks(range(scores.shape[1]), trg_words_list)\n",
    "        plt.show()\n",
    "\n",
    "    def __call__(self, sentence):\n",
    "        # 预处理句子，标点符号处理等\n",
    "        sentence = preprocess_sentence(sentence)\n",
    "        encoder_inputs, attn_mask = self.src_tokenizer.encode(\n",
    "            [sentence.split()],\n",
    "            padding_first=True,\n",
    "            add_bos=True,\n",
    "            add_eos=True,\n",
    "            return_mask=True,\n",
    "        )  # 对输入进行编码，并返回encoder_inputs和encoder_padding_mask\n",
    "        # 转换成tensor\n",
    "        encoder_inputs = torch.Tensor(encoder_inputs).to(dtype=torch.int64)\n",
    "        # 由输入得到预测结果，\n",
    "        # 注意，这里输入的样本只有一个，符合Seq2Seq模型的生成方法要求batch_size=1\n",
    "        preds, scores = self.model.infer(encoder_inputs=encoder_inputs, attn_mask=attn_mask)\n",
    "\n",
    "        # 通过tokenizer转换成文字\n",
    "        # trg_tokenizer.decode方法要求输入的是字符串列表，所以需要[preds]\n",
    "        # 方法返回字符串列表，所以需要[0]\n",
    "        trg_sentence = self.trg_tokenizer.decode([preds], split=True, remove_eos=False)[0]\n",
    "\n",
    "        # 对输入编码id进行解码，转换成文字,为了画图\n",
    "        src_decoder = self.src_tokenizer.decode(encoder_inputs.tolist(), split=True, remove_bos=False, remove_eos=False)[0]\n",
    "\n",
    "        self.draw_attention_map(\n",
    "            # [1, sequence length, sequence length] \n",
    "            # -> [sequence length, sequence length]\n",
    "            scores.squeeze(0).numpy(),\n",
    "            src_decoder,  # 注意力图的源句子\n",
    "            trg_sentence,  # 注意力图的目标句子\n",
    "        )\n",
    "        return \" \".join(trg_sentence[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3aa03683664d5b62",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:37:15.036459Z",
     "start_time": "2025-01-27T04:37:15.036459Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:11:39.762322Z",
     "iopub.status.busy": "2025-01-27T13:11:39.762059Z",
     "iopub.status.idle": "2025-01-27T13:11:39.914714Z",
     "shell.execute_reply": "2025-01-27T13:11:39.914160Z",
     "shell.execute_reply.started": "2025-01-27T13:11:39.762301Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAGkCAYAAACsHFttAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAjztJREFUeJzs3XmYXGWZ8P/vqX2v7qre9ySdzr7vBEgg7IIIghsguIDL6Kg4OoPzG0UdB4dXZvTVdxxwZhRFVJBVdgkkJED2fe/u9L7vte/n90eTSipdHRLoqiY19+e66pI6dZ+nnrtP97nP85znREVVVRUhhBAiB2kmuwNCCCFEpkiRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEzpIiJ4QQImdJkTvF2rVrURQFRVHYs2fPpPShubk52YeFCxdOaNtr167l61//+oS2mctqamr46U9/OtndSFJVlbvuuguXy3XG31FFUXj66aez2rcPmnvvvXfC/35y2Qfh3JepPkiRO82dd95JV1cXc+fOTSk4iqJgMBiora3ln//5nzn9X0M7ePAgH/vYxygsLMRoNFJXV8d3v/tdAoFAStzevXv58Ic/TFFRESaTiZqaGj7+8Y/T29sLQGVlJV1dXXzzm9/MWs7i/PDSSy/xm9/8hueeey75O5pOV1cXV199dZZ798Hyd3/3d6xfv36yu3FeOdO579TXli1bkvsEg0G+973vUVdXh9FopKCggJtvvpmDBw+mtB0IBLjnnnuYNm0aJpOJwsJC1qxZwzPPPJOMefLJJ9m2bduE56Wb8BbPcxaLhZKSkpRtr776KnPmzCEcDrN582Y+//nPU1payuc+9zkAtmzZwmWXXcZll13G888/T3FxMdu2beOb3/wm69ev5/XXX8dgMNDX18e6deu49tprefnll8nLy6O5uZlnn30Wv98PgFarpaSkBJvNlvXcxQdbY2MjpaWlXHDBBWk/j0QiGAyGMb+//xvZbDb5GzpHZzr3ncrtdgMQDoe57LLLaG1t5YEHHmDFihX09PRw3333sWLFCl599VVWrlwJwBe/+EW2bt3Kz3/+c2bPns3AwABvvfUWAwMDyXZdLhcej2fiE1NF0po1a9Svfe1ryfdNTU0qoO7evTslbt26deqXv/xlVVVVNZFIqLNnz1aXLl2qxuPxlLg9e/aoiqKoP/7xj1VVVdWnnnpK1el0ajQafde+fO9731MXLFjwvvI53Zo1a9SvfvWr6re+9S01Pz9fLS4uVr/3ve8lP3/ggQfUuXPnqhaLRa2oqFC/9KUvqV6vN6WNzZs3q2vWrFHNZrOal5enXnHFFerg4KCqqqoaj8fVf/mXf1FrampUk8mkzp8/X3388ccnrO9f+cpX1K997WtqXl6eWlRUpD700EOqz+dT77jjDtVms6nTpk1TX3jhBVVVVfXXv/616nQ6U9p46qmn1NN/5Z999ll16dKlqtFoVN1ut/qRj3wk+Vl1dbX6ox/9SP3MZz6j2mw2tbKyUn3wwQdT9t+3b596ySWXqCaTSXW5XOqdd9455mc2EW6//XYVSL6qq6vVNWvWqH/zN3+jfu1rX1Pdbre6du1aVVVVFVCfeuqprPfxbL344ovq6tWrVafTqbpcLvVDH/qQ2tDQkPx869at6sKFC1Wj0aguWbJEffLJJ1P+Ds/m2Gbi7yeXne2571Q//vGPVUVR1D179qRsj8fj6tKlS9XZs2eriURCVVVVdTqd6m9+85t37cfZfO+5kunKc7Rjxw527tzJihUrANizZw+HDh3i7rvvRqNJ/XEuWLCAyy67jD/84Q8AlJSUEIvFeOqpp8ZMd2bLww8/jNVqZevWrdx///384Ac/4K9//SsAGo2G//t//y8HDx7k4Ycf5rXXXuPb3/52ct89e/awbt06Zs+ezdtvv83mzZu57rrriMfjANx333389re/5T//8z85ePAg3/jGN7j11lvZuHHjhPW9oKCAbdu28dWvfpUvfelL3HzzzVxwwQXs2rWLK664gttuu23MFPF4nn/+eW644QauueYadu/ezfr161m+fHlKzAMPPMDSpUvZvXs3X/7yl/nSl77E0aNHAfD7/Vx55ZXk5+ezfft2Hn/8cV599VW+8pWvTEi+p/rZz37GD37wAyoqKujq6mL79u3A6M/EYDDw5ptv8p//+Z9j9stmH8+W3+/n7rvvZseOHaxfvx6NRsMNN9xAIpHA5/Nx7bXXMnv2bHbu3Mm9997L3/3d301aX8X4Hn30US6//HIWLFiQsl2j0fCNb3yDQ4cOsXfvXmD03PfCCy/g9Xqz39EJK5c5YLyrGbPZrFqtVlWv16uAetdddyVj/vjHP57xyuNv//ZvVbPZnHz/ne98R9XpdKrL5VKvuuoq9f7771e7u7vH7JepkdyFF16Ysm3ZsmXq3//936eNf/zxx1W32518/8lPflJdvXp12thQKKRaLBb1rbfeStn+uc99Tv3kJz/5Pns+tu+xWEy1Wq3qbbfdltzW1dWlAurbb799Vlf7q1atUm+55ZZxv7O6ulq99dZbk+8TiYRaVFSk/vKXv1RVVVUfeughNT8/X/X5fMmY559/XtVoNGmP6fv17//+72p1dXXy/Zo1a9RFixaNieOUkVy2+/he9PX1qYC6f/9+9cEHH1TdbrcaDAaTn//yl7+UkVyGvdu579TXCSaTKWWfU+3atUsF1D/96U+qqqrqxo0b1YqKClWv16tLly5Vv/71r6ubN28es5+M5CbJn/70J/bs2cPevXt57LHHeOaZZ/iHf/iHlBj1LEdmP/rRj+ju7uY///M/mTNnDv/5n//JzJkz2b9/fya6Psb8+fNT3peWliYXvbz66qusW7eO8vJy7HY7t912GwMDA8mR0YmRXDoNDQ0EAgEuv/zy5P0Qm83Gb3/7WxobGye871qtFrfbzbx585LbiouLAZL5vJsz5ZPuOxVFoaSkJNn+4cOHWbBgAVarNRmzevVqEolEcrSXaUuWLDnj5x+EPp6uvr6eT37yk0ydOhWHw0FNTQ0Ara2tHD58mPnz52MymZLxq1atmpR+ipPnvlNfpzrb897FF1/M8ePHWb9+PTfddBMHDx7koosu4oc//GEGep1KitxZqKyspLa2llmzZnHzzTfz9a9/nQceeIBQKERdXR0wejJJ5/Dhw8mYE9xuNzfffDM/+clPOHz4MGVlZfzkJz/JeB4Aer0+5b2iKCQSCZqbm7n22muZP38+TzzxBDt37uT//b//B4wuaAAwm83jtuvz+YDRKcBT/yAOHTrEn//854z1/dRtiqIAkEgk0Gg0Y/4Ao9Foyvsz5XOm70wkEufU70w6tXidL6677joGBwf51a9+xdatW9m6dStw8vfs3ZzNsRUT48S579TXCXV1dWc8752IOUGv13PRRRfx93//97zyyiv84Ac/4Ic//OFZH/f3Sorce6DVaonFYkQiERYuXMjMmTP593//9zEnv7179/Lqq6/yyU9+cty2DAYD06ZNS66unCw7d+4kkUjwwAMPsHLlSurq6ujs7EyJmT9//rjLsmfPno3RaKS1tXXMH0VlZWU2UkhRWFiI1+tN+bmefhV6pnzOxqxZs9i7d2/Kd7z55ptoNBpmzJjxntudSB+0Pg4MDHD06FH+v//v/2PdunXMmjWLoaGhlP7u27ePUCiU3HbqknU4u2MrMu8Tn/gEr776avK+2wmJRIJ///d/Z/bs2WPu151q9uzZxGKxlGOdCVLkzsLAwADd3d20t7fz4osv8rOf/YxLLrkEh8OBoij893//N4cOHeKjH/0o27Zto7W1lccff5zrrruOVatWJR/Afu6557j11lt57rnnOHbsGEePHuUnP/kJL7zwAtdff/2k5lhbW0s0GuXnP/85x48f53e/+92YhQz33HMP27dv58tf/jL79u3jyJEj/PKXv6S/vx+73c7f/d3f8Y1vfIOHH36YxsZGdu3axc9//nMefvjhrOezYsUKLBYL3/nOd2hsbOTRRx/lN7/5TUrM9773Pf7whz/wve99j8OHD7N//37+9V//9ay/45ZbbsFkMnH77bdz4MABXn/9db761a9y2223JadOJ9sHrY/5+fm43W4eeughGhoaeO2117j77ruTn3/qU59CURTuvPNODh06xAsvvDBmluNsju354Be/+MW7TpdPthPnvlNfJ4rSN77xDZYvX851113H448/TmtrK9u3b+ejH/0ohw8f5r//+7+Tsytr167lwQcfZOfOnTQ3N/PCCy/wne98J3kezSQpcmfhsssuo7S0lJqaGu666y6uueYa/vSnPyU/v+CCC9iyZQtarZarr76a2tpa7rnnHm6//Xb++te/YjQagdErF4vFwje/+U0WLlzIypUreeyxx/iv//ovbrvttslKDxhdCfpv//Zv/Ou//itz587l97//Pffdd19KTF1dHa+88gp79+5l+fLlrFq1imeeeQadbvRxyx/+8If80z/9E/fddx+zZs3iqquu4vnnn2fKlClZz8flcvHII4/wwgsvMG/ePP7whz9w7733psSsXbuWxx9/nGeffZaFCxdy6aWXntPDqBaLhZdffpnBwUGWLVvGTTfdxLp16/jFL34xwdm8dx+0Pmo0Gv74xz+yc+dO5s6dyze+8Q3+z//5P8nPbTYbf/nLX9i/fz+LFi3iH//xH8dceJzNsT0f9Pf3T9j96kw5ce479XXiX9MxmUy89tprfPrTn+Y73/kOtbW1XHXVVWi1WrZs2ZJ8Rg7gyiuv5OGHH+aKK65g1qxZfPWrX+XKK6/ksccey3gOinq2dw7/F1i7di0LFy78QPxTTvfeey9PP/20TMOI//Wam5uZMmUKu3fvln+qK0M+KOe+TBxrGcmd5j/+4z+w2WxZW+14utbWVmw2G//yL/8yKd8vhPjfabLPfVdfffWYf11lIshI7hQdHR0Eg0EAqqqqMBgMWe9DLBajubkZAKPROCmLNoT4IJGRXOZ9EM59meqDFDkhhBA5S6YrhRBC5CwpckIIIXKWFDkhhBA5S4pcBoTDYe69917C4fBkd2XCSE7nB8np/JCLOcEHMy9ZeJIBHo8Hp9PJyMhIxp/mzxbJ6fwgOZ0fcjEn+GDmJSM5IYQQOUuKnBBCiJylm+wOZEsikaCzsxO73Z78R0MzxePxpPxvLpCczg+S0/khF3OC7OWlqiper5eysjI0mjOP1f7X3JNrb2+Xfz1ECCFySFtbGxUVFWeM+V8zkrPb7QBcZLgBnaJ/l+jzx8Anx///azpfaXLw//9SF/jg/B+tThRdKDevj/X+2GR3YcIZenyT3YUJFYuH2djw/5Ln9TP5X1PkTkxR6hR9ThU5rcE02V2YcJrMziZPCl00B4tcPDeLnE6Xe0VOp83BK0c4q1tPsvBECCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ+kmuwPng7bYUZpjh4kQxKbkM1O/FKemYNz4nngLDbF9hFQfFsVOrW4RhdpyABJqgsbYXvoTHQRUHzoMuDUl1OoXYlIs2UqJvoOb6d27gWjQi9lVRsXqG7AWVY0bP3R8L13bXyTiG8LoKKBsxbU4q2YlPx9u2kf/obcJ9LcTDweYcePdWArKs5FKUu/hzfQceCen/DKqVt6AtfAMOTXtpWP3OznZC6hYei3OypM5DTXvo+/o2wQGRnOa9eG7sbizmxNAd/2bdB7dSDTkxZJXypRFH8HmHj+vgba9tB14mbB/CJO9gKr515BfejKvwfb99DS+jX+og1gkwLzLv441P7t5dTa9RUfDRiJhL1ZHKdPmXY89f/yc+jv30XLkZUKBIczWAmpmX42reDSnRCJOy5GXGeo5QigwgE5nwlk4nZrZV2M0ObOVEu3tb9PWuolIxIfVVkJd3XU4HJXjxvf27qfp+F8JhYYxm91Mm3YV7oIZyc8PH/oz3d27UvZxuaazYOFnMpbD6VoHd9I0sJVIzIfdWMTM0ivIM5eljfWF+qjv24Qn1E0oOsKM4nXUuJe/rzYnwnkxklu7di1f//rXJ+W7u+PNHI3tYqpuHisM12DX5LMr8joRNZQ2fjjRx/7om5Rrp7HCcA2Fmkr2Rt/AlxgGIE4MjzrIFN08VhquYYHhYvyqhz2RjVnLaahxNx1vP0vJkiuYceM3MLvLaHzhIaJBb9p4X3cTzesfwT1zBTNvvBtnzVyaXvk1wcGuZEwiGsFaMoWyFR/KVhopBo/vpn3bs5QuvIJZH/4GFlcZ9a+cIaeeJo5vfISC6SuY9eG7yaueS+NrvyY4dEpOsQi24ilULJ2cnAD6W/fQsvcvVMy5fLQY5ZVx+I3/IhrypY339jdTv+VRiqYsZ/4VX8dVNodjbz5MYKQ7GROPRbAXTKFq/jXZSiNFX8cemg7+haoZl7FozdewOks5sOW/iYTT5+QZbObIzkcprlrGojVfw106h8PbfovfM5pTIh7BP9xBZd06Fq75GjOXfZqgr4/DW3+TtZx6evbRUP8CNTXrWLrsb7DZStm759dEIulzGhlp4dDBP1FaupSly75CQeFs9u9/BJ+vOyXO5arjgtX3JF+z53wiG+kA0DVyiCM966ktvJBVUz+L3VTMzpY/EY7508bH1SgWQx51RWsx6KwT0uZEOC+K3JNPPskPf/hDAGpqavjpT3+ate9uiR2hQltLuW4aNo2TWbrlaNHSEW9MG98aO4JbU0qNbjY2jZNa/QIcSj6t8aMA6BUDSwzrKNFWY9U4yNMUMFO/DK86SFDN3IE+Ve++N3DPXIl7xnLM+SVUXvRRNDo9A0e3pY3vO7AJR+UMihdcgim/mLJlV2MuKKfv4JvJGFfdUkqXXIG9vC4rOZyu5+AbFNStpGD6csx5JVRd8E5O9elz6j20CWf5DErmXYI5r5jyxVdjcZfTe/hkTu7apZQtvAJ76eTkBNB17A2Kpq6gaMoyLM5ipiy5EY1OT29T+ry66jeTVzKDsplrMTuKqZx3Fda8crrrT+ZVWLOEijmX4yienq00UnQ0bqKkagXFVcuw2IupnX8jWq2entbtaeM7j28mv6iOitq1WOzFVM+8ElteOV1Noznp9GbmXnAnheULsNiKcLiqmTbvI/hGOggFhrKSU1vbZsrKllFatgSrtZgZM65HozHQ1bkzbXx721u4XNOpqr4Yq7WIqVMvx24vo6N9S0qcRqPFaLQnX3q9ORvpANAysI2KvAWU583HZixgdulVaDU6Oob3pY13msuYUXwppc7ZaJT0k4Tn2uZEOC+KnMvlwm63Z/17E2ocrzqIS1OS3KYoCi5NCSOJ/rT7jCT6cWlKU7a5NWXjxgPE1AgAegwT0OszS8RjBPrbsVecPMEpigZ7eR2Bnpa0+/h7WsYUL0fFDPw9zZns6llLxGMEBtpxlJ2WU2kdvt70Ofn6WrCXnZZT+Qz8vc2Z7Oo5ScRj+Ic6cBan5uUsmo5vYJy8BlpS4gGcJXXjxmdbIhHDN9JBXmFtcpuiaMgrmI53KH0fvUOt5BWk5pRXWIdnqHXc74nHQoCCLgtFIZGI4fN2ku9KzcnlmobHk76PIyOtKfEwOhU5clr88HATmzf9iC1b/o2jR58mGg1MfAJpJNQ4nlA3buuU5DZFUXBbaxgOdHxg2jwb50WROzFduXbtWlpaWvjGN76BoigoipLR740QRkXFoJhSthsUE2E1mHafMKG08eNNb8bVOPWxPZRoatAp+onp+BnEQ35QE+jNqRcNOrONaCD91F4s6EVntp0Wbyc2zlRgtsXCoznpTstJb7aNO10ZC3rRn56TyT5u/GSIRd45VsbUfupNNiKh9P2MhrzoTafH24mOE59t0WROpx0r4/g5RUJeDKf9DAxG27g5JeJRmg69QGH5AnR6U9qYiRSNBlDVBAbDaT93g41wZJycIj4M+tNyMtiIhE/Gu9zTmTXrZhYu+hzTpl3F8FATe/f8BlVNTHwSp/cvFkBFxahLXSdg0FmJxNJPwU5Gm2fjvFp48uSTT7JgwQLuuusu7rzzzjPGhsNhwuFw8r3H48l0985ZQk2wL7oJUJmlH3uDVghxbhKJOEd2PAKoTJt/42R3530pLl6Q/G+brQSbrYQtb/+EoaHjuE4bBYrxnRcjuRNcLhdarRa73U5JSQklJSXjxt533304nc7kq7Jy/FVO4zFgREEZMwqLqCGMSvppECNjR20Rdezo7kSBC6l+FhvWZWUUB6A1WUHRjBmxxII+9Jb0U8KjozbfafHeMSOnyaIzjuZ0+sgyGvSNGbEm9zHbiZ6eU8g7bvxk0BneOVanLciIhnwYTOn7OTpqOz3ei36c+GzTJ3M67ViFx8/JYLKPWZQSCfvG5HSiwIWCw8xddWdWRnEAer0FRdGMWWQSjfgwGsbJyWAjEj0tp4gPg3H842Q2u9DrLQSDA++/0+/CoLOgoBCOpU6PRmJ+DDrbOHtlv82zcV4VuXNxzz33MDIykny1tbWdcxsaRYtdcTGYOLniSVVVBhPd4z5C4NQUpMQDDCS6UuJPFLiA6mWJYR0GxXjOfXuvNFodloIKvB31yW2qmsDbWY+luDrtPtbi6pR4AG/HMazFNZns6lnTaHVY3BV4uk7LqaseW1H6nGyF1Xi7UnPydBzDWlSTya6eE41WhzW/nJGehuQ2VU3g6W3A5h4nL3c1Iz2peY301I8bn20ajQ6bs5zh/tSchvsbsOen76M9vyolHmC4rx7HKY8cJAucv595q+4cLaZZotHosNnLGBpKzWloqBGHI/1jEU5nFUODqYvXBgcbcI4TDxAKjRCNBjEaHBPT8TPQKFocphIG/c3JbaqqMuBvIc/y3h43yUSbZ/W9GWt5khmNRhwOR8rrvajWzaQj3kBn/Di+xAiHY9uIE6dMOxWAA5G3qI/uTsZX6WYykOikOXYYf2KExug+POogVdrR519OFDhPYpB5+tWoqITVIGE1SEKNv//Ez0LR/IsZOLKVgWPbCQ310LbpCRLRCO660SnT5tcfpXPb88n4wrkX4Wk7Qs++DYSGe+ja8TKBvnYK56xOxsRCAQL9HYSGegAIj/QS6O8gGsjONHHxnIvpP7aVgfrtBId7aH3rCRKxCO7pozk1vfEoHTtO5lQ0+yJG2o/Qc2A0p87dLxMYaKdo1ik5hQMEBjoIDY/mFBrpJTCQvZwASusupvf4VvqadxD09NC080nisQiFU5YB0LD1D7Tue+Fk/PQLGek+SufRjQQ9vbQdeAX/UDsl01Pz8g91EPSM5hX09uEf6iASzE5e5dMuortlGz2tOwh4e2jc9xTxeITiyqUAHN31R5oPvZiML5t6IcO9R2lv2EjA20vLkVfwDbdTOmU0p9EC9zt8w+3ULf4kqqoSCXmJhLwkErGs5FRZeSFdnTvo6tqF39/LsaPPEI9HKC1bDMChQ4/T2PhyMr6i8gIGB4/R2roJv7+XpuOv4vV2UF6xEoBYLExDw4uMjLQSDA4xONjA/v2/w2x24XJnZ1VstXs57cN76Bjehy/cz6Gul4gnopTnzQdgf8dfONazIRk/urCkB0+oB1WNE4758IR68EcGz7rNTDiv7skBGAwG4vHsFAOAEm0NETVMY3QvYULYlXwWGy5JTleGVD9wcgFMnqaQefrVNMT20hDbg0Wxs0B/MTZNHgBhNUBfoh2ALZEXUr5rif4yXNrijOeUP20RsaCfrh0vEwt4MLvLmXbNncnpyqhvOGVRj61kCjXrbqVr+4t0bXsBo7OQKVd8BrPr5CrSkZYDtG78U/J98/pHAChZfAWlS6/MeE6uqYuIhfx07n6ZaNCD2VXO9CvuTE4/Rvyn5VQ8halrbqVj14t07HwBo6OQaZd+BnP+yZyGWw/QsvlkTk0bR3MqXXgFZYsynxNAQdVCYmE/bQdefudh8DJmXvz55NReODAMp+RlL6ihduWnaDvwMm37X8RkK6Bu9e1YnCen9gc7D3J8+2PJ9w1bfg9A+ezLqZx7RcZzKixfSDTip/XoK+88DF7G3JWfO5lTMPVYOVw1zFjyKVoOv0TLkZcwWwuYtfzTWB2jOUVCIwx2HwJgz8afpnzX3Au+QF7BtIznVFw8n2jUT9PxV4lEvNjspcxf8BkM70xXhkPDKKecJ5zOambP+TjHj/+V442vYLG4mTfvVmy20ZwURYPP10131y5isRBGo51813SmTr0MjSY7p+1S52wi8QANfZsIx/w4jEUsqfoYxneegQtGPZx67gtHvbx9/H+S75sHttI8sJV8SxXLa245qzYzQVFVVc1Y6xNk7dq1LFy4kJ/+9KdcccUVmM1m/uM//gOj0UhBwfj/8sipPB4PTqeTS4wfy9r9r2zo//Tiye7ChNNEJ7sHE0/vz/yKuGzThT7wp473RO/LzugvmwzdH4zVtRMlFg+z/ui/MTIy8q6zdOfddOUPfvADmpubmTZtGoWFhZPdHSGEEB9g58V05YYNG5L/vXLlSvbu3Tt5nRFCCHHeOO9GckIIIcTZkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs3ST3YFsU6MxVEWZ7G5MmGBB7uRygjY82T2YeMEC7WR3YcKVvz4y2V3ICM1IYLK7MPH6Bye7BxNLjZx1qIzkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicpZvsDpwP2hL1tKhHiBDCRh4zNItxKu60sR2JRrrUZnyMAODAxTTNvGR8Qk3QqO6nX+0iiA8delxKMdOVBRgVc9ZyGtqxmYGtrxP3eTEWl1F8xQ2Yy6rTxg7vfpuR/TsI93cDYCqpoHDtNSnxR/7l7rT7Fl56Le6Vl058AmkM7N7MwI7Xifm9mArLKLn0Biyl6XMCGDm6h943XyLqGcSQX0DxRddinzo7+Xk8EqZ303N4Gg4QD/kxONy4Fl+Ea8EF2UgnaXDXZga2j+ZlLCqjdN0NmM+Ql+foHno3v0R0ZDSvojWpefW++RKeI3uIeodRNFrMxRUUXnQNlnGOfya09W6jufstIlEfNksJMyuvxmkrHze+Z/AgDZ2vEwoPYzG5qS2/jMK86cnPDzQ9TdfA3pR93I5pLK67NWM5nK51eBdNQ9uJxP3YDUXMLFpHnql03Phu71EaBjYTjI1g0edTV7CGQutUABJqnPqBzfT7jxOMjqDTGHBbqplesAaTzpatlGgNHaIpvJ9IIohd62KmZRV5usJx47sjTTQEdxJM+LBoHNRZllGor0x+vt//Bp2R+pR93LpyltqvylgOUuTeRXeilWPqHmYpS3AobtrUY+xObOQCzTUYFNOY+CF6KVaqmKEUoEFLs3qY3YmNrNRchUmxkCCGVx1iqjIbm5JHjAhHE7vZo25ihfaKrOTkObSb3vXPUHzVzZjLqhjc/gZtf3yIqV/4B3RW+5j4QGsjjjmLMVfUoGh1DG55jbY/PMiUu76N3p4HQO3f3puyj6/xCN3P/wn7jAVZyAhGjuymZ+MzlF52M+bSKgZ3vkHLEw8x/bP/gM6SJqeOJtqff4Tii67BPnUOw0d20fbMr5l6292YCkZPTD0bnsHfVk/FNbegd7jwtRyl69Un0FkdOGrnZi+vDc9QevloXgM736Dl8Yeo/dw4x6qjifa/PELRxddgnzaHkcO7aHvq10z99N2YCkfzMuYXUrLuRgx5bhKxKIM7NtL6+IPU3vkddJbMn0C7Bw9wtO0VZlV/CKe1gtaeLeyqf4TVc7+CQW8dEz/sa2P/8SeorVhHgbOO7sH97G38IytnfwGbuSgZ53bUMmfK9cn3GkWb8VxO6PIe4Uj/BuYUXo7TVErL8E52djzOhdWfw6gbm9NQsIN93X9hesHFFFqn0eU9zO7Op1hV9WnsxkLiiRjeUA/TXKuwG4uIxkMc6XuN3Z1Psqrq09nJKXKcI8GtzLGsxqkrpCV0kJ2+l7jQcRNGzdgL8qFYD/v8rzPdvJRCfRVdkUZ2+15lleN67FpXMq5AV8Fc60XJ9xoye5xkuvJdtKpHKVemUqaZik1xMlNZihYdnWpT2vi5mlVUaqZjV/KxKg5mK8tQURlUewDQKQYWa9dSrKnCqjhwKgXM0CzGyxAh1Z+VnAa3bcS5cCV5C5ZjLCyh5Oqb0Oj0jOzdlja+7PpbyV+yGlNxOcaCYkqu+TioKoHmk1dkOpsj5eWrP4CluhZDfvoR70Qb2LmR/HkryZ+7HJO7hNLLb0Kj1zO0P31OA7s2YZsyk4Jll2J0F1O8+mpMxeUM7t6cjAl0NuOcvQxrZS0GpwvX/FWYCssIdrdmJSeAgR0byZu/krx5yzEWlFB6xWhewwfS5zW48528lo/mVXTh1ZiLyxk6JS/n7CXYauow5LkxFZRQfMn1JCIhQn2dWcmppWcLFQWLKS9YhM1cyKzqa9Fq9HT0704b39qzFbezlpqS1djMhdSWX4rDUkprb+rPQKPRYtTbki+9LnszIy1DO6hwzKfcOQ+bsYDZRVegVfR0eA6kjW8d3kmBZQpT8pdjM7iZ7r4Qh6mY1uHRn4Fea2Rpxccosc/EanCRZy5jVtE6POEeglFPdnIKHaDCOINyYx02bT6zLavRoqMjcix9TqGDFOgrmGKaj02bx3TzEhxaN62hwylxGkWDUWNJvvQaY0bzkCJ3Bgk1jpchXEpxcpuiKLiUYobV/rNqI04cFRW9Mv6BjBEFQIfh/XX4LKjxGKGudqw1dcltiqLBMqWOYEfzWbWRiEZQE3G0Jkvaz2M+L76GQzgXLp+ILr97f+Ixgj3tWKtSc7JW1RHsak67T7CrGVvV9JRttuqZBE6Jt5TV4G08SNQ7jKqq+FvriQz1YauZkYk0xlDjMULd7VirT8uruo5AZ3PafQKdzVirU/Oy1swcN16Nxxja+zYaowlTYdlEdX1ciUQcr78Tl2NqcpuiKLgcUxnxt6fdZ8TflhIPo1ORI77U+CFvMxv2/B/e3P8LDrc8RyQWmPgE0kiocTzhbtyWk9O9iqLgtlQzHEp/4TAc6sRlSZ0eLrDUjBsPEEuEATJeFOCdnOL9uHUnfycURcGtL2M41pt2n+FYLy5d6u9Qgb6C4Xhq/GCsm9eHf8+mkT9zyP8mkURo4hM4xXk3XfnnP/+Z73//+zQ0NGCxWFi0aBHPPPMMVuvYKYH3K0oEFRUDqdOSBkz4OburqQZ1L0ZMuChO+3lcjdOQ2EeJUoVO0b/vPr+bWMAPamLMVJfOaicwkP6X93R9rz+HzubEMqUu7ecj+7ejMRixz5j/vvt7NuLBcXKy2AkMps8p5veOmcbUWe3E/N7k+5JLb6Tzr49x7KEfgEaDoiiUXf4xrBXTJj6JdH08kdfp/bTYCZ8przTH9tS8ALyNB2n/y+9Qo1F0NjvVN38xK1OVkVhg9G/qtGlJg86KP5T+wjEc9WE4bcrPoLcRifqS7wuctRTlz8JsyCMYHqKhYz27j/2e5bM+h6Jk9lo+Eg+iomLUpl70GXQW/IHBtPuEY/4x05gGrZVIPP1sTjwR41j/G5TaZ6HTZr7IRdTQaE6nTUsaFDP++EjafcJqME28iUji5MVGgb6cYn01Zq2dQNxDfXAnO30vs9J+XcaO03lV5Lq6uvjkJz/J/fffzw033IDX62XTpk2oqjomNhwOEw6Hk+89nuwM8U/VnDhMt9rGEs0laNPcH0ioCfYn3gJUZipLs96/92LgrfV4Du2m6ta/QaNLX5RH9m7DMWfJuJ+fLwZ3byLY1ULVRz6H3pGPv72RrvVPorM5sVWnL/DnC2tlLdNu/yaxoJ/hfVto/8tvmXLL19Le5zsflLhO3iO1W4qxWYp5c///ZdDbjPu0UeD5JqHG2dv9LCoqswsvn+zuvC+lhpMXiHatC7vWxSbP4wzGunHrMzOTcF5NV3Z1dRGLxbjxxhupqalh3rx5fPnLX8ZmG3sFet999+F0OpOvysrKNC2emR4DCgoRUofTEUJjRnena0kcoVk9zGLNGuxK3pjPTxS4EH4WadZmZRQHoLNYQdGMubJPNwI43cCW1xl4ez2Vn/wipqL0v5CB1uNEBnvJW7hiwvr8brTmcXIKjJ+TzmonFhj/Z5CIRujd/AIla6/HPm0OpsIy3IsuwjFjIQM7Xs9MIqf38URep/fz3fI6i2OrMRgx5BdiKauh7KpPoCgahvdvndgE0jDoLKN/U9HUEUsk5seoTz+SNOptRGKnxUd9GMaJB7AY89HrLATD6UdSE8mgNaOgEI6nTo9GYoExI9ATjDor4dNzivsxaFPjE2qcvV3PEox6WFr+sayM4mB0BKagEE4EU/uoBjGkWXQCYFTMaeJDGDTpb2sAWLQO9IqJQCJzg5DzqsgtWLCAdevWMW/ePG6++WZ+9atfMTQ0lDb2nnvuYWRkJPlqa2s75+/TKFrs5CcXjQCo6ugikjylYNz9mhOHOa4eYpHmYhyKa8znJwpcAC+LNWsxnOF+3URTtDpMpRX4T1k0oqoJAs31mMtrxt1v4O3XGHjzr1R+4i7MpeNfMAzv3YqppAJT8fjLwSeaRqvDXFyBvzU1J39rPebSmrT7mEtr8LWmLmX2tRzD8k68mkigJuKgKCkxikZJO3OQCYpWh6mkAn/LaXm11GMpq0m7j6WsJuXnAOBvOTZu/Ml2VRLx2Pvt8rvSaLTYrWUMeo+nfPeg5zhOa0XafZzWSgY9qQu9BjzHcdrSxwOEIh6isQAGfeZHphpFi8NYwmCgJblNVVUGgi3kmdJfDOaZyhgMpi5gGgikxp8ocIHoMMvKP4ZBm72FNBpFi0NbwGCsK7lNVVUGop3k6YrS7pOnK2IwlnpPcSDaQZ42fTxAKOEnqoYy+vjUeVXktFotf/3rX3nxxReZPXs2P//5z5kxYwZNTWNXOhqNRhwOR8rrvahSZtCpHqcz0YRf9XBE3UGcGKXKFAAOJLbQkNiXjG9OHKZRPcBszTJMWAmrQcJqkJg6urgkoSbYl3gTD4PM1axERU3GJNT4e+rjuXItX8PIni2M7NtOuL+Hnhf/TCIawTl/dKFI57OP0vv6c8n4gbfX0//Gi5R86OPonS5iPg8xn4dEJJzSbjwcwntkL86FK7OSx6ncS9YwtH8Lwwe3Ex7ooevV0Zzy547m1P7io/RsOpmTe/FF+JqP0L9jA+GBHnrfeolQTxuuRRcCoDWasFRMo2fjX/C3NRAZGWDowDaGD+3AUTsve3ktXcPwvi0MH3gnr1dG88p7J6+O5x+l542TebmWXISv6QgD29/J682XCHa3kf9OXolImJ43nifQ2UxkZJBgdxudL/6RmG8Ex4yFWcmpunglHX276Ozfgy/Yx+GW54gnopQVjH7/gaanqG9/NRlfVbyCAU8Dzd1v4Q/209ixAU+gk6qi0Z9BLB7hWNsrDPvaCYaHGfAcZ0/DH7EYXRQ4snP/tDp/Ke2efXR4DuCLDHCo9xXiiSjljtFp1P3dz3Os/42TOeUtod/fRPPQdnyRARoG3mQk1E1V3iJgtMDt6XoWT7iHeSUfQiVBOOYjHPNl7TxRbZpLe/goHeF6fPFhDgXeJE6McsPoVP1+/0aOBbefzMk0h/5oO82h/fjiwzQEdzES76fKNAuAmBrlaGAbw7FegnEvA9FOdvv+ikXjoEA//gXL+3Ve3ZOD0RU+q1evZvXq1Xz3u9+lurqap556irvvTv8w8vtVoqkimghzXD1AWA1hJ49FmjUY33lGLqQGUE652m9XG1A5ca/tpCnKHKYpcwkTpJ/Rq52tiVdSYhZrLsHF+Fc9E8UxexHxgI++N14i7vdgLC6n8uN3obONXvVGPUMpI5ihXW+hxuN0PvlwSjvuC6+g8OKTD3F6D+0GVcUxe1HGczidc+YiYkEfvW++RCzgwVRYTvVH70pO00U9QynHyVI+hYprbqX3zRfp3fw8hrxCKq//TPIZOYCKa2+jd9PztL/wCPFQAL3dRdHqa8jP4sPgzpnvHKs3XyLm92AsKqfqplPy8qYeK0v5FCquvZXeTS/Su+l5DPmFVN7wmeQzcmg0RAZ7aX9mO/GgH63Jiqm0kppPfgVTQUlWcipxzSUSC9DYuYFw1IfdUsLi6bckpytD4RHgZE55tkrmTbmRho7Xaeh4DYvRxYJpn0g+I6coCr5gL50De4nFQxj1dtyOaUwrvwSNJjunuFL7TCLxAA0DbxKO+3EYilhSflNycUkw5k3JKd9czvySa6kf2MSxgU1Y9fksKrsBu3H0QetwzEefvwGAt1tT/+6WlX8cl6Uq8zkZphJJhGgI7SScCOLQulliuzK5uCSY8KXmpCtmvvUS6oM7ORbcgVXjYJHtsuQzcgoK3vggnb56omoEo8ZCga6cWvOSjD7TqKjZmnuZAFu3bmX9+vVcccUVFBUVsXXrVm699Vaefvpprr766jPu6/F4cDqdrNXcmLX7X9nQ/vfZu/eVLdrwu8ecb9Tzas7k7JS/nn6V3flOM5KdRw+yqj/z9yazKaZGWD/8O0ZGRt51lu68Gsk5HA7eeOMNfvrTn+LxeKiuruaBBx541wInhBDif6fzqsjNmjWLl156abK7IYQQ4jyRg5MoQgghxCgpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLN9kdEO9P9ZO9k92FCdf24aLJ7sKEi1kmuwcTr2eFc7K7kBHFb092DyaepndgsrswsVT1rENlJCeEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWbrI7cD5oS9TToh4hQggbeczQLMapuNPGdiQa6VKb8TECgAMX0zTzUuJ71XbaEw14GSJKhBWaK7Ar+VnJ5YTWoZ00DWwlEvdjNxYxs/hy8sxlaWPbhvfQOXIAX7gPAIephOmFa1Li93c+R6fnQMp+busUllZ+PHNJnGZw12YGtr1OzO/FWFRG6WU3YC6tThs7tPdthg/uINzXDYC5pIKii68ZEx8e6KFnw3ME2hpR1QRGdzGVH7kDvSN7x2t422YG33yduM+LsaSMwqtvwFyRPq9wbzcDr79IqLOd2MgQhVdeT/6qNantbX+T4e1vERseBMBQVIJ7zRVYp8/KeC4n9O/fTN/u14kFvJjcZZRffAOW4vQ5AQw37KFn60tEvIMYnQWUrLoWR81sANR4nO6tL+BtOUzYM4jWYMJWWUfpqg+htzqzlRJtfdto7nmLSNSHzVzCzMqrcVrLx43vGTpIQ+frhCLDWIxuassvo9A5/ZTPD9PevwNvoItoPMjKmV/AbinJRipJreFDNIUPEFGD2LX5zDStIk9XOG58d7SJhtAuggkfFo2DOtNSCvWVyc97os20RY7giQ8QVcOssl2PQ5v+XDpRZCT3LroTrRxT9zBVmcNyzRXYlTx2JzYSUUNp44fopVipYonmEpZpLsOomNmd2EhIDSRj4mqMPKWQWmV+ttJI0eU5zJHe16gtuJBVNZ/BbixiZ9ufCMf8aeOHAq2UOmazrOpTrKj+NCa9g51tfyIU9abEFVinsrb2K8nXgrLrs5EOACOHd9Pz+jMUrr6SqbffjamwjJbHHiLm96aN97c24py1mJpPfJkpt/4tOnseLY89SNQ7nIyJDPXT/PufY3QXUf3JLzPtjr+jcNXlKNrsXRt6D+ym7+VncK+9kqov3I2xuIyORx4i5kuflxqNoM93U3DZtWht9rQxOkceBZd9iKov3E3VXd/AMmU6HX/4H8K93ZlMJWm4fjddm5+heNmVTP/Y3ZgLymj6y0PEAuMcq64mWl95hPxZy5n+sW/imDqPlhd/TWigC4BELEKwr4OipVcw/WN3U331HYSHeml+/r+zkg9A9+ABjra/wtTSNayY+QXs5mJ2NTxCJJr+b2rY18b+picoL1jEiplfoDBvBnuP/xFfsDcZE09EyLNVUVt+WbbSSNEVOc6R0DZqTQtZZfswdo2Lnf6XCSeCaeOHYj3sC2yg3FDHKtv1FOmr2B1Yjzc+lIyJqzHytMXUmZZmK40PfpGLRqOT+v2t6lHKlamUaaZiU5zMVJaiRUen2pQ2fq5mFZWa6diVfKyKg9nKMlRUBtWeZEyppoapmjm4lOxelZ3QMriNCucCyvPmYzMWMLvkKrQaPR0j+9LGzy/7MFX5i3GYirEZ3cwtuRoVlYFAc0qcRtFi1NmSL73WlIVsRg3s2Eje/JXkzVuOsaCE0itvQqPXM7x/W9r4iutuxbVoNabicozuYsqu+jioKv6W+mRM76YXsE2dRfHa6zAXV2DIL8A+fS46a/rikQlDb2/EsXglzkXLMRaVUHTtTSh6PZ7d6fMylVdReMWHccxbNG4xts2Yg61uNgZ3IYaCIgrWXYPGYCDU3pzBTE7q27MR15yVuGYtx+QqoXztTSg6PYOH0+fUv28T9qqZFC2+FJOrmJIVV2MuLKd//2YAtEYzU6//InnTF2LKL8JaUkP5xTcS7Gsn4h1K2+ZEa+ndQkXBYsrdi7CZC5lVde3o39TA7rTxrb1bcTtqqSlejc1cSG3ZpTjMpbT2nfwZlLkXMK10DW771KzkcLqWyAEqDDMoN9Rh0+Yz27waraKjI3IsbXxr5BAFugqmGOdh0+Yx3bQEh9ZNa+RQMqbMUEutaRFuXfpZo0yY0CL30EMPUVZWRiKRSNl+/fXX89nPfhaAZ555hsWLF2MymZg6dSrf//73icViyVhFUfjlL3/Jhz/8YaxWK//8z/9MbW0tP/nJT1La3LNnD4qi0NDQMJEppEiocbwM4VKKU/rnUooZVvvPqo04cVRU9IoxU908Jwk1jifUjdtak9ymKApuSw3DwY6zaiOeiKKqCfRac8r2wUArr9f/XzYdf4hD3S8Tiae/4ptoajxGqLsda01dcpuiaLBW1xHobD6rNhLRCGoijtZkGW1TTeBrPIzBVUjLYw9y9Bff5fjvfoqnfn8mUkhLjcUIdbZjnXpKXhoN1ql1BCeoIKmJBJ79u1GjEUwVNRPS5pkk4jGCfe3YKlKPlb2ijkB3c9p9At3N2Cqnp2yzVc4cNx4gHgkBClqjedyYiZJIxPEGOnGdUowURcFln8qIvz3tPiP+NlyO1OLldkwbNz7bEmocT3wgpRgpioJbV8ZwvC/tPsOxXlynFa8CXTnDsd608dkyoUXu5ptvZmBggNdffz25bXBwkJdeeolbbrmFTZs28elPf5qvfe1rHDp0iAcffJDf/OY3/OhHP0pp59577+WGG25g//79fO5zn+Ozn/0sv/71r1Nifv3rX3PxxRdTW1ubti/hcBiPx5PyOldRIqioGEgdkRgwESH9dOXpGtS9GDHhovjdg7MgEgugomLUWVO2G3RWIuNMV57uWN8GjDobbktNcluBbSrzSq9laeUnqCtcy2CglZ1tj6GqifEbmiCxgB/UBDpL6ghLZ7WPO115ut6Nz6GzOZOFMu73kYiG6d/6GrYpM6m++Qs4ps+j/anf4G/N3IXVqeLv5HX6tKPWaic+znTl2Qr3dFL/o3+g/offpve5xyn9+GcwFmV+ZiEeGudYWexEx5mujAW86Mxj48eb3kzEonS//Rx50xehNWR+NuHE35Qhzd9UOOpLu0845hsbr7cRGSc+2yJqePQ8oaReJBgUM5FTbr2cKqwGMSqmNPHZudgdz4QWufz8fK6++moeffTR5LY///nPFBQUcMkll/D973+ff/iHf+D2229n6tSpXH755fzwhz/kwQcfTGnnU5/6FJ/5zGeYOnUqVVVV3HHHHRw9epRt20aH8tFolEcffTQ5Okznvvvuw+l0Jl+VlZXjxmZKc+Iw3Wob8zUXolW0Wf/+TDg+8DZdnsMsKr8RrebkdFipYzZF9unYTUUU2+tYXHEznlAXg4HWSezt2enfsp6RI7up/Mhn0Oj0AKiqCoC9dg7uZWswFZdTsHIdtmmzGdrz9mR2d0IY3EVUf/GbVN35NZzLLqDn6T9k7Z5cJqnxOC0v/xZVVSlfe9Nkd0d8AEz4PblbbrmFJ554gnA4DMDvf/97PvGJT6DRaNi7dy8/+MEPsNlsydedd95JV1cXgcDJq4OlS1NvSpaVlfGhD32I//mf/wHgL3/5C+FwmJtvvnncftxzzz2MjIwkX21tbeecix4DCsqYUVuE0JjR3elaEkdoVg+zWLMGu5J3zt+dKQadBQVlzCKTSMw/5srydE0DW2ka2MLSyo9jNxWdMdZiyEOvNROIZP6eiM5iBUUz5so+5ve+6/2z/m2v0791PdU3fxFT0cmpFp3FChoNRnfq6MboLiKapfs82nfyOn3UFvd7x11UcrYUnQ6DuxBTWSWFl12LsbiM4a1vvK82z4bWNM6xCnjRW8ZZKGOxEwuOjT99NDha4B4m6h1k6vVfzMooDk7+TZ0+ExKJ+THqbWn3MepsY+OjPgzjxGebQTGOnidOG4VF1CAGxZJ2H6NiJnzagrzR+MxPGZ/JhBe56667DlVVef7552lra2PTpk3ccsstAPh8Pr7//e+zZ8+e5Gv//v3U19djMp38hbRax55sP//5z/PHP/6RYDDIr3/9az7+8Y9jsaT/YQMYjUYcDkfK61xpFC128lMWjajq6CKSPKVg3P2aE4c5rh5ikeZiHIrrnL83kzSKFoephEF/c3KbqqoMBFrIM4+/3LlpYAvHB95iSeXHcJpL3/V7QlEP0XgQoy7zf7SKVoeppCJl0YiqJvC31GMpqxl3v/6tr9H/1l+puvkuzKWpI31Fq8NcUkVkMPV+QmSoL2uPDyg6HaayCgJNp+SVSBA4Xo95gu+fqaqKGotPaJvpaLQ6zIUV+NpTj5WvvR5LSU3afSwlNSnxAL72YynxJwpceKSfqdd/CZ3pzBdsE0mj0WK3lDHoPX6yP6rKoPc4TmtF2n2c1koGPamL1wbOEJ9tGkWLQ+tmMNaZ3KaqKgOxTvK06R8hyNMVpcQDo/G6M18QZ9qEr4U2mUzceOON/P73v6ehoYEZM2awePFiABYvXszRo0fHvY92Jtdccw1Wq5Vf/vKXvPTSS7zxRuavOgGqlBkcUrfiSLhwKm5a1aPEiVGqTAHgQGILJizUakYfB2hOHKZRPcBczUpMWJNXQlp06JTRqbCoGiZEgDCjn/nV0atUA6Yxc+CZUO1azoGu53CYS3GaSmkZ2kE8EaHcOZrD/s6/YNTZqStaC8DxgS009G9iful1mPVOwrHR+wZajQGdxkAsEaGxfzPF9hkYtVYC0WGO9b6ORZ9PgXVKxvMBcC9dQ+cLf8BcUom5tIqBHRtJRCPkzVsOQMfzj6KzOShecy0A/VvX07f5JcqvvRWDw0XMN3rPVmMwojGMLhJyL19L+7O/w1I5FWtVLb6mI3gbDlHzyS9nJSeA/FVr6H7qDxjLKjGVVzG8ZTQvx6LRvLqefBSdw0HhZaN5qbEY4b7RizI1HifmHSHU1YHGYMDgHj059b36HNbaWeid+SQiITz7dxFsbsR1211Zyalw4Rra1v8Bc1EllqIq+vduJBGLkD9rNKfWVx9Fb3VQumo0p4L5F9H49P+jb/cG7DWzGK7fTbC3jYq1NyfzbHnpNwT7O6j50OdQEwmi/tHjqTVZ0GThkY/qopUcbHkah6UMh6Wc1r4txBNRytwLATjQ/BRGvZ3p7zwOUFW0gh3HfkNzz1sUOuvoHjyAJ9DJ7Krrkm1GY0FCkZHkozr+0OhiN4PeNu4IcUJzMszlQHATDm0BTm0hLZGDxNUY5YbR+9b7AxsxaqzJxwGqDLPZ7n+B5vB+CnSVdEePMxLvZ7Z5dbLNSCJMSPURTozO3Pnjo88TGxUzRs34g5b3IyNH/5ZbbuHaa6/l4MGD3Hrrrcnt3/3ud7n22mupqqripptuSk5hHjhwgH/+538+Y5tarZY77riDe+65h+nTp7Nq1apMdH2MEk0V0USY4+oBwmoIO3ks0qxJ3mANqQEURUnGt6sNqCTYn3grpZ0pyhymKXMB6FM7OaSeXCp8QH0b1NSYTCp1zCISD9DQt4lw3I/DWMSSyo8nF6MEox7gZE5tQ7tQ1Th7O59OaWeaezW1hRehoOAN99E5coBoPIRRZ6PAOoXawovRaLLzTJlz1iLiQR99m18i5vdgLCqn6ua7ktOVUc8QnHKchna/hRqP0/7MwyntFFxwBUUXXgWAo24+pVfcxMCW9XSvfwqDq4jKj9yBpSJ7S7rtcxcR8/sYeP0l4j4PxpJyym+9C90705WxkaGU37+Y10Prgw8k3w+9tYGhtzZgrp5G5Wf+BhhdVNP91KPEfR40RjPG4lLKb7sL67QZWckpb/oiYkEfPVtfIhbwYCooZ8q1dyWnK6Pe1JyspVOouvxWure+SPeW5zHkFVJ99WcwuUdnFKL+ETzNBwGo/9MDKd819SNfxlZ+7hfV56rENZdILEBj1wbCUR92cwmLa29JFqNQZIRT/6bybJXMm3IjDZ2v09D5GhajiwVTP4HNfHLU0zdylIMtzyTf729+YjSnkjVMK1ub8ZxKDVOJqCEaQrsIq0EcWhdLrFdg1IxeiAcT/pSc8nXFzLespT60k2OhnVg1DhZZ1mHXnpz56Iu1ciC4Kfl+X3ADANOMC6k1Lc5IHop64g77BEokElRUVNDV1UVjYyNTp548Kbz88sv84Ac/YPfu3ej1embOnMnnP/957rzzztEOKQpPPfUUH/nIR8a0e/z4caZNm8b999/Pt771rXPqk8fjwel0slZzY3JElQu007MzUsqmtg9P7vRGJsQyc5E6qUzpV5Kf94rfHpnsLkw4TeO5r0n4IIupEdZ7HmFkZORdb0Vl5DJbo9HQ2dmZ9rMrr7ySK6+8ctx9z1RzOzo60Ov1fPrTn37ffRRCCJH7zot/uzIcDtPX18e9997LzTffTHHxB+OZMyGEEB9sH/h/1gvgD3/4A9XV1QwPD3P//fdPdneEEEKcJ86LInfHHXcQj8fZuXMn5eXjL3MXQgghTnVeFDkhhBDivZAiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJylm+wOZF0iDkru1Ha1s2eyuzDhirc7JrsLE67tS7HJ7sKEi+2wTXYXMkJJJCa7CxOvuGCyezCx4mHwnF1o7pzthRBCiNNIkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpZusjuQzh133MHw8DBPP/30uDFr165l4cKF/PSnP814f9rUBlo4RoQQNpzMYBFOxZU21qeO0MghvAwRIkAdC6hSpqfEqKrKcQ7SRSsRQhgxU0o1U5iFoigZzwegNXKY5vABImoQmyafWeaVOLWFaWPbI0fpjDbiiw8B4NC6mW5aMibeFx+mPryDoVg3CVRsmjwWWC7BrLFlPB+A9va3aWvbRCTiw2otoa7uOhyOyrSxnZ3b6e7ehd/fA4DdXs7UqVekxEciXhobX2ZwsJ5YLEReXg3Tp1+HxVKQlXxOGH5pK8N/eZP4sA9DdTGFn/0QptqKd93P++Z+en72ONalMyn99qeS21VVZfCx1/Cs30nCH8I0s4rCz1+HodSdyTRSDO7azMC214n5vRiLyii97AbMpdVpY0P93fRtfpFQdztRzxDFl16Pe+ma99VmJrT2bae5920iMR82czGzyq/CaS0fN757+BANXRsIRYaxGF1ML1tHoePkuaJn+DDtA7vwBLqIxoOsrLsTh6UkG6kktQ7vomlwG5G4H7uxiJmFl5FnLh03vtt7hIb+zQRjI1j0+dQVrKHQNg2AhBqnvn8T/f7jBKMj6DQG3JYaphdejElnz1gOMpJ7F91qG8fYx1Rms5zLsJPHbjYRUUNp4+PEsWCllnkYMKWNaeYI7RxnJotYxZXUMo8WjtFGQyZTSeqOHudoaBvTjAtZaf0wdq2Lnf5XCCeCaeMHY92U6Kew1HoVK6wfwqSxstP/CqGEPxkTSHjYHngBq8bJUuvVXGC7nqnGBWjQZiWnnp59NDS8QE3NOpYu/RtstlL27v01kYgvbfzw8HGKixewcOHnWbz4ixiNTvbu/TXh8AgwWgj273+EYHCQefNuY9myr2Ay5bFnz/8Qj0eykhOA96399P/2JVw3raXyX7+IsbqEzh/9lthI+rxOiPYO0f+7lzHNGnuSH35mMyMvbqXwzuuo+Je70BgNdP7otyQi0UylkWLk8G56Xn+GwtVXMvX2uzEVltHy2EPE/N608Wo0gsHppmjNteis6U+G59rmROseOsjRzr8yreRiVs64E7u5mJ3HHyUc9aeNH/a3sb/5ScrdC1k5406KnDPY0/QY3mBvMiaeiJJnrWR62bqs5HC6Lu9hjvS9Tq17NauqbsduLGRnx2OEY+lzGgp2sK/rL5Q757Gq6g6KbNPZ3fkU3nAfAPFEDG+4h2nuC1hV/WkWlt2APzrI7o4nM5qHFLl30coxyplCmVKDTXEwk8Vo0dJJc9p4p+JiujKfEqUSzTg/3hEGKKSMAqUUs2KlWKnARTEehjKYyUnN4YNU6OsoN0zHps1jtukCtIqOzmh92vj5ljVUGWbh0LqxavOYY1qNispgrCsZ0xDaRYGugjrTMhxaNxaNgyJ9FUaNOSs5tbVtpqxsGaWlS7Bai5kx43o0GgNdXTvTxs+e/XHKy1dit5dhtRYxc+aNqKrK0FAjAMHgAB5PGzNmXI/DUYHFUkhd3fUkElF6evZmJSeA4efewrluCY5LFmOoKKLwzutQDHq8r+8adx81kaDn53/G/bFL0Bflp36mqgy/8Db5N16MbdksjNUlFH3lRuJDXvzbj2Q6HQAGdmwkb/5K8uYtx1hQQumVN6HR6xnevy1tvLm0iuJLPoxz1iIUbfrJp3Ntc6I1922hwr2IcvdCbKZCZld8CK1GT+fgnrTxLX3bcDtqmVJ0ATZTIbWll+Awl9LWvz0ZU+aaz7SSi3HbpmQlhzF9HNpBhWM+5c552IwFzC66Eq2ip8OzP21869AOCqxTmOJagc3oZnrBRThMxbQOj/6u6rVGllZ8nBL7TKwGN3nmMmYVXYYn3EMw6slYHhkrcolEgvvvv5/a2lqMRiNVVVX86Ec/AmD//v1ceumlmM1m3G43d911Fz7f+Femfr+fT3/609hsNkpLS3nggQcy1e3UHNQEXoZxUZTcpigKLooZZuA9t+vEzSC9+NXRq0yvOswI/bjJ/FREQo3jTQzg1pUltymKgktXynC89wx7nhQnjkoCvWIERk+cfbE2LBoHO/0v87r3D2zx/YXeaEtGcjhdIhHD5+skP782uU1RNLhc0/B4Ws+qjXg8iqrG0eksyTYBNJqTJ1VF0aDR6BgZyU5eaixG+HgX5nnTTvZBo8EybxqhY+3j7jf45w1oHTYcly4Z81msd4j4sA/L/JNtai0mjLXlhI61TWwCaajxGKHudqw1dcltiqLBWl1HoLP5A9PmuUgk4ngDXSnFSFEUXLYpDPvTH6cRf/uY4uW2Tx03PtsSahxPqBu3tSa5TVEU3NZqhoOdafcZDnXistSkbCuwTBk3HiAWDwOg1xjfd5/Hk7Eid8899/DjH/+Yf/qnf+LQoUM8+uijFBcX4/f7ufLKK8nPz2f79u08/vjjvPrqq3zlK18Zt61vfetbbNy4kWeeeYZXXnmFDRs2sGvX+FeyAOFwGI/Hk/I6V1HCqKhjph0NGImQfrrybNQwk2IqeZuXWa8+wVZepZLplCpV77nNsxVR38lJSR1hGRXzuNOVpzsW2oFRseDSlb7TZpA4MZrC+ynQVbDEcgXF+mr2BF9jMNY94TmcLhoNoKoJDIbUe396vY1w+OymqxobX8JgcJCfP3ryt1gKMRrzaGx8mWg0SCIRo6VlI+HwyFm3+X7FPQFIJNDmWVO2a/OsxIbT9yF4pAXPa7so+sKH034eGx69mNQ6U39WOqeN+PCZp0AnQizgBzWBzpI67aiz2t/z1GIm2jwXkXhg9G9Kn/ozNeqthGPpf6bhmA+DPvW4GvQ2IuNMBWbbiZyMWkvKdoPWSiSevo/hmH9svG78+HgixrH+jZTaZ6HTZq7IZWThidfr5Wc/+xm/+MUvuP322wGYNm0aF154Ib/61a8IhUL89re/xWodPci/+MUvuO666/jXf/1XiouLU9ry+Xz893//N4888gjr1o3OTT/88MNUVJz5xvt9993H97///Qxk9/710E43rcxlBTYceBnmGHsxqibKlJrJ7t4ZNYX30R09zjLr1WiV0V8f9Z3PinRVVBvnAKOLU4bjvbRHjuDSZfdm+blqadlIb+8+Fi36PFqtHgCNRsu8ebdw5MiTbN78QxRFQ37+NFyuundpbfIkgmF6fv4ERV/4MFqH9d13EGKSJNQ4e7ueQUVldtEVGf2ujBS5w4cPEw6Hk0Xp9M8WLFiQLHAAq1evJpFIcPTo0TFFrrGxkUgkwooVK5LbXC4XM2bMOGMf7rnnHu6+++7ke4/HQ2Vl+pV249FjREEZM2qLEB53UcnZqGcfNcygRBntjw0nQTVAM0cpo+Y9t3s2DMo7Oampo7awGnzX+2fN4f00hfezxHoldu3J1aUn2rRpnSnxVo3zrKdA3w+93oKiaMYsMolGfRiNZ1611dq6idbWjSxY8FlsttRVY3Z7OcuWfZVYLEQiEcNgsLFjx3/gcIy/Ym4iaR0W0GiID6deCceH/ejyxuYV7Rkk1jdM178+enKjOnoJ0vCJe6n+6d+iyxsdbcRHfOjyT7YRG/FhrBl/1dxE0VmsoGiIBVJHWDG/d9xFJZPR5rkwaC2jf1PR1N+/cNSPUZd+ZbFRZyNy2qKUSNSHQffBuDg5kVM4HkjZHon7MWjT99Gos46Nj42NT6hx9nY+SzDqYVnlJzI6ioMMTVeazdlZbHAmRqMRh8OR8jpXGkWDnTwGOXmiVlWVQXrJ470vt04QRyH1UYHR92r6HSaQRtFi17gZOGXRiKqOLiLJ0xaNu19TeD/Hw3tZbLkcpzZ1Cb1G0eLQFuBPpE4JBxIeTErmHx/QaHTYbGUMDZ1cnaqqCYaGGnE4xp8Cbml5g+bm15g//w4cjvFnBnQ6EwaDjUCgH6+3g4KC2RPa//EoOh3GqaUEDxxPblMTCQIHjmOqG9tffVkBlT/5Gyrv/1LyZV0yA/OcGirv/xK6Age6ony0eTYC+0+2mQiECDd0YKo7t4vA95STVoeppAJ/y8lFTqqawN9Sj6Ws5gPT5rnQaLTYLaUM+JpP+X6VQV8Tedb0v1dOawUDvqaUbQPe8eOzTaNocZhKGAycvP+sqioDgRbyzGVp98kzlaXEAwwEmlPiTxS4QHSIZRUfx6DNfK3ISJGbPn06ZrOZ9evXj/ls1qxZ7N27F7//5FXMm2++iUajSTs6mzZtGnq9nq1btya3DQ0NcezYsUx0fYwq6uikiU61Gb/q4Qi7iBOj9J0R1wF1Gw3qydVGCTWBVx3Gqw6TIEGYIF51mIB68iqvgFKaOEK/2kVQ9dOrdtDKMQpJ/8sz0WqMc+iIHqMjUo8vPszh0FvE1Rhl+tFndPYH36A+tCMZ3xTeR0N4F3PMF2LW2AgnAoQTAWLqySXnNYZ5dEebaI8cJZDw0Bo5RF+sjUrDzKzkVFl5IV1dO+jq2oXf38uxY88Qj0coLV0MwKFDj9PY+HIyvqVlI01Nf2XmzI9iMuUTDnsJh73EYuFkTG/vfoaGjhMMDtLXd4i9e/+HwsLZuFzTx3x/puRdewGe9TvxbNhNpL2Pvv96DjUcwb52NK+eXzxB/6N/BUBj0GOsKk55aawmNCYjxqpiFJ0ORVHIu2YVQ09uxL/jCOHWHnp+8STafDvWZdk5Vu6laxjeu4XhA9sJD/TQ9cqfSUQj5M1bDkDH84/Ss/G5ZLwajxHq6SDU04EajxPzjhDq6SAy1HfWbWZaTeFKOgZ20TG4F1+oj8PtLxBPRClzLQBgf8vT1HeePB9WFy5nwNNIc+/b+EP9NHRtxBPspLJgWTImGgviCXTje2cJfiA8gCfQTTia+XunANX5S2kf2UvHyAF84QEO9b5CPBGl3DFvNKeu5znWtzEZX5W/lH5/E82D2/BFBmjo38xIqJuqvNHf1YQaZ0/nM3jC3cwrvRaVBOGYj3DMR0KNZyyPjExXmkwm/v7v/55vf/vbGAwGVq9eTV9fHwcPHuSWW27he9/7Hrfffjv33nsvfX19fPWrX+W2224bM1UJYLPZ+NznPse3vvUt3G43RUVF/OM//iMaTXaefihRKomqYY5ziDAh7DhZxIUYldHpyhCBlFFZmCBbeTX5voVjtHCMPApYyloAZrCQRg5yhN3Jh8HLmcpUsjNCKNFPJaKGaAzvJqwGsWtcLLZckZyuDCX8KJqTObVFjqKSYG/w9ZR2phoWUmtaBECxvprZ6iqaIvs4EtqKVeNkgfkS8nVjj2kmFBfPJxr109T0KpGIF5utlPnzP4PBMDpdFQ4Ppzxo39m5FVWNc/Dgoynt1NRcypQplwGjD4M3NLxAJOLDYLBTUrKImppLspLPCfYL5hH3BBh87DViwz6MNSWUfee25LRjtH8EzvEfEMi7/kIS4Qi9Dz5LIjD6MHjZd25DY9BnIoUxnLMWEQ/66Nv8EjG/B2NROVU335WcWox6hlJyivo8HH/45Irqge0bGNi+AUvlNGo++Tdn1WamleTPIRIL0Ni1kXDMh91czOKpn8L4zmKUUMSTcp7Is1Yyr+YGGrpep77rdSxGFwunfAy7+eRsSu/IMQ62PZt8v69l9HmyqcUXU1s69mH4iVZqn0UkFqRhYDPhuB+HsYgl5TdjfGdKNRjzpBynfHM580uvpb5/E8cGNmHV57Oo7AbsxtF/NCIc89HnH51tebvlNynftaziE7gsmVl4p6iqmpE5skQiwX333cevfvUrOjs7KS0t5Ytf/CL33HMP+/fv52tf+xpvv/02FouFj370o/zbv/0bNtvoL8Tp/+KJz+fjS1/6Ek8++SR2u51vfvObPP/88+f0L554PB6cTidruR6dkp0/5mzQ2LPzR5xN0SXZGyllS9uXYpPdhQln2JGdf8km2ypeyc7zqtmkBMLvHnQeicXDrG/8GSMjI+96KypjRe6DRorc+UOK3PlBitz5439zkZN/8UQIIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs3ST3YFsU4xGFEU/2d2YMIrFPNldmHD6LYcmuwsTbsrItMnuwoSrvz0x2V3IiMH5eZPdhQnnmaJMdhcmVDwcgh+fXayM5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM7STWRja9euZeHChfz0pz+dyGYnXVvsKM2xw0QIYlPymalfilNTkDbWlximMbYPT2KQEH7qdEuo1s0cExdSA9RHdzOQ6CROHItiY7Z+FU6NO9PpANDq30+Tfw+RRAC73s1M+0XkGYrTxvqig9T7tuGJ9hFKeJlhX02NdcH7ajNTWmNHaY4dIqKOHqtZhmXjHqv2WD2d8eP4EiMAODQupusXpsQ3RPfSHW8hpPrRoMWhcVGrX0jeOG1mQlvfNpp73iIS9WEzlzCz8mqc1vK0se39O+ka2Icv1AuAw1JKbdm6MfG+YB/1na8y7G0hQQKbqZD5Uz+G2eDMeD4Ank1vMvLaBuIeL4byUtwfvQFjdVXaWP/e/Yz8dT3R/n6Ix9EVFuK8ZA22ZUuSMUMvvox/1x7iw8MoWh2GygryP3QVxprqrOQD0HtkM90HNxANerG4yqhcfgO2gvQ5BYe76djzEoGBdiL+ISqXXk/x7IvHxEUCI7TvfI6RjiMk4hFM9gJqLvgE1oLKTKcDwPC2zQy+9TpxnxdjSRmFV9+AuTz9zzTc283AhhcJdbYTGxmi8MrryV+5JiWmf8NLDG58JWWb3l3ElK/8Q8ZymNAil4u6480cje1ilm45Tk0BrfEj7Iq8zmrjdRgU05j4OHHMio1ifRVHozvTthlVw2wPv4JLW8wiwyUYMBFQvegxZDodALqC9RzxvskcxxqchmJa/PvYOfQcFxZ8EqPWMiY+ThSL1kGJaRpHvG9OSJuZ0B1r5mh0J7P1K3Bq3LTEjrAz/BqrTR/GmOZYDSZ6KNHWkKcvRIuWpthBdobXc4HpOkzKaJ+tioNZ+mWYFRsJ4rTEDrMrvJ4LTdenPf4TntPgAY62v8Ksqg/htFTQ2ruFXQ2PsHr2VzDorWPih7wtlLjmkmetRKPR0dz9JrsafseqWV/GZHAAEAgPsuPYrylzL2Ja6Vp0WiO+YB9aJTunA/+uPQw+9Szuj30UY00Vng2b6Pnlryj/x2+jtdvHxGssZpyXr0NfXISi0xI8cJj+R/+E1mbDPGsGAPrCQtw33YDO7UaNRvFseIPuX/6Kin/6B7Q2W8ZzGmzaTduOZ6leeRPWgip6Dm+i/tWHmHv936M3j80pEYtgtLlxVS+gbcczaduMhQMcefHn2EtqmX7ZneiNVkLefrRGc6bTAcB7YDd9rzxD0YduxlRRxfCWN+h45CFqvvIP6Kxjc1KjEfR5bmyzF9L38tPjtmsoLKHi019Mvlc0mZ1Q/EBPV0YikcnuAi2xI1RoaynXTcOmcTJLtxwtWjrijWnjnRo3dfrFlGhr0KBNG9McO4RJsTBHvwqnpgCzxoZbW4pFM/YXJxNaAnupsMym3DILm87FbMcatIqOjuCRtPFOfTEzHBdQap4+bk7n2mYmNMcOn3Ks8pitX4EWLZ2xhrTx8w0XUqWbgUPjwqpxMke/EhUYjHcnY0p1U5LHxqbJY4Z+CTGieBNDWcmppXcLFQWLKXcvwmYuZFbVtWg1ejoGdqeNnzflRioLl2G3lGA1FTC7+jpUVWXQ25SMaeh8jQLndOoqLsdhKcVidFGUNyNt0cyEkQ0bsV+wAvvK5RhKSnB/7KMoBj3eLdvTxpun12JdMA9DSTH6ggIcay/CUFZK6PjJnGxLF2OeUYe+wI2htATXDR9GDYWIdHRlJaeew29QMH0lBbXLMeeVUL3yo2i0evobtqWNtxZUUbn0OlxTFqFo0l9cdB94DYM1jymrP4GtoAqj3Y2zbAYme3ZmEYa2bMSxeCXORcsxFpZQdO1NKHo9nt3pczKVV1F4xYdxzF2Eoh3/gknRaNDZHMmX1pLZi5AJL3KJRIJvf/vbuFwuSkpKuPfee5OfDQ8P8/nPf57CwkIcDgeXXnope/fuTX5+7733snDhQv7rv/6LKVOmYDKZzmq/TEmocbzqIC5NSXKboii4NCWMJPrfc7t9iXYcGjd7I5vYEPozW8Iv0D7OiXiiJdQ4nmgfbkNFcpuiKLgNFQxHu8+wZ3bbfC998KqDuLWlKX1waUsZPstjFSeOSgK9kn5EnVDjtMca0KHHrsmfkH6fSSIRxxvoxGWfmtymKAou+1RG/O1n1UY8EUVVE+h1o1f/qqrSP1KPxehiV/0jbNj3f9h65L/oHc7OxYgaixFp68BUV5fcpmg0mOqmE25ueff9VZXg0Xqivb2Ypk1NHxOL4X1rC4rZhKG8bML6Pp5EPIZ/oB1H6fTkNkXR4Citw9/37jmNZ7j9EBZ3JY0bH2bPY9/j4F8eoO/Ylono8rtS4zFCne1Yp55ynBQN1ql1BNub31fbkcF+Gh+4l6af/TNdTz5CdCSzF4wTPj/x8MMPc/fdd7N161befvtt7rjjDlavXs3ll1/OzTffjNls5sUXX8TpdPLggw+ybt06jh07hsvlAqChoYEnnniCJ598Eq12dNRwNvudLhwOEw6Hk+89Hs855xIhjIo6ZlrKoJjwJ869vROCqo/2+DGqtLOYYpiDJzHA0dgONIqGMm36P9yJEkmEUFExalKnEA1aM/7Ie/tly0Sb59yHE8eK1GNlVEz437nn9m6ORXdjVMy4NKUp2/vi7eyLbCZODCNmlhjXZWWqMhILjOakSx1hGXRW/KGzK9z1Ha9i1NuThTIS8xNPRGjqeZPa0kuYXn4Z/Z4G9h7/E0um347LXjPRaaSI+/2QSKC1p169a+12or294+6XCAZp++4PUWMx0Ghw33wj5pl1KTGBA4foe/gR1GgUrcNOyZfuQmvL/Og0FvaDmhgzLakz2wh5xs/p3YS9A/QdfYvi2WsonbsO/0AbrdufQtFqKZi27P12+4zigdGctKdNS2qtdiL97z0nc3k1xus/gaGgiJjXw8DGV2j79S+o+dK30Bgz8zc14UVu/vz5fO973wNg+vTp/OIXv2D9+vWYzWa2bdtGb28vRqMRgJ/85Cc8/fTT/PnPf+auu+4CRqcof/vb31JYWAjA5s2bz2q/09133318//vfn+j0JoQKOJTRRQ4wuuDBp47QHqvPeJET6TVFD9Adb2aZ8XK0SuqUbL6mhFXGDxEhREesgb2RTawwXp32Pt8HSVP3ZrqHDrB0+h1o35kSU1UVgCLnDKqLVwFgt5Qw7G+jvX9nxovce6UYjZR9+24S4TChY/UMPv0sOrcL8/TaZIxp+jTKvn03cb8f31tb6fvN7yi9+2/T3uc7P6hY3BVULL4GAIu7guBwN31H3854kcsU6/RZyf82Fpdhqqim6ac/xHtwD87FKzPynRM+XTl//vyU96WlpfT29rJ37158Ph9utxubzZZ8NTU10dh48v5WdXV1ssABZ73f6e655x5GRkaSr7a2tnPOxYARBYWIGkrZHlFDGJX3fvPXiAmrJnUVm1VxEFL977nNs2XQmFBQCCcCKdsj8SAGzXtbIJKJNs+5DyeOFanHKnwWx6o5eoim2EGWGNelnYbUKTosGjt5mkLmGFahQUNHFqaXDTrLaE6x1N+LSMyPUX/m+xjNPW/R3LOZxbW3YbecXOE62qYGq6kwJd5mKiAUObsR7/uhtVpBoyHu9aVsj3u9aO2OcfdTNBr0hQUYK8pxXroW64L5jLz6WkqMxmhEX1iAqaaagk99DDRavFvS3z+aSDqjFRQN0aA3ZXss6ENveu8FVm92YHamrk42OYuJ+DM/O6K1jOYU96fmFPd70dom7qJBazKjdxcSGXzvt3/ezYSP5PR6fcp7RVFIJBL4fD5KS0vZsGHDmH3y8vKS/221pk4vnO1+pzMajcmR33ulUbTYFReDiW6KtKNLdlVVZTDRTaVuxntuN09TSOC06c6A6sWkZH5qRaNocegLGYx0UGwaHTWqqspApJ0qy7wPTJvvpQ92xcVA/LRjFe+mSlc37n5N0YM0xQ6w2HjpWT++oaKSID4h/T4TjUaL3VLGoPc4RXmjj6GMLiI5TmXh8nH3a+5+k6buTSyafitOa+o9KY1Gi8NaRiA8kLLdHxrMyuMDik6HobKc0LF6rPPnAqAmEoSONWC/aPXZN6Sqo1OX7zdmAmi0OqzuCrxd9eRXzXvnqxN4uuspmnEOOZ3GVlhDyNOXsi3k6cNgy/z9YEWrw1RWQeB4PbaZJ3MKHK8nb/mFE/Y9iUiY6GA/uvlL3j34PcraIwSLFy+mu7sbnU5HTU1NxvebKNW6mRyMvo1D48ahuGmNHyFOPDmteCDyFkbFzHT9ImB0cYJfHb0iTpAgrAbwJgbRok+unqzSzWJ75GWaYgco1lQzovbTHq9ntn5FdnKyLODAyGs49IU49UW0+PcRV2OUm0dPpPuHX8WotVJnX5XMyRcbvXpUiROO+/FE+9Eqeqw651m1mQ01ulkciL6FI+Yafdwjdpg4Mcp000bziryJSbEkj1VT9CANsb3MN1yIWbERVoMAaNGhU/TE1BhNsf0UaiswYiZKmNbYMcJqgBJtdp6/qi5aycGWp3FYynBYymnt20I8EaXMvRCAA81PYdTbmV5+2WhO3Ztp7NrAvJobMRvyCEdHR0xajQGddnRBTU3xBexr+jN5tipctin0exroHznKkro7spKTc+0a+n7/R4xVFRiqqvBs3IQaiWBfMToF1/fIH9A5neRfNzpNN/zX9RgrK9EVuFFjMYKHDuPbvhP3xz4KQCIcZuSV9ZjnzUHnsBP3B/BuepPYyAjWhWOf58yE4lkX0/TmH7EUVGJ1V9Fz+A0SsQgFtaMXI02bH0VvcVKx+EOjfY7HCI30AKAm4kQCIwQGO9DojJgco6sni2dfzJEXf07X/lfJr16Iv7+V/votVK+8KSs55a9cQ/fTf8BYVompvIrhLRtJRCM4Fo7m1PXUo+jsDgovu3Y0j3iMcN87OcXjxDwjhLo70BgMGFyjMwd9rzyLtW42+jwXMe8IAxteRtFosM9dnLE8slbkLrvsMlatWsVHPvIR7r//furq6ujs7OT555/nhhtuYOnSpRO630Qp0dYQUcM0RvcSJoRdyWex4ZLkFNjoFKOSjA+rQbZEXky+b4kfpiV+mHyliKXGy4HRxwwW6C+mIbaH47H9mBUbM3RLKdVOyWguJ5SapxNJhGjwbiOcCODQF7Ak/9rk82zBuC81p7iftwceS75vDuyhObCHfH0Zy90fOas2s6FEV0OEMI2xfYTV4OixMl6acqyUU/Jqix9DJcHeyBsp7UzVzaNWvwAFBX/CQ2fsDSKEMWDEoXGzzHgFNk1ednJyzSUSC9DYtYFw1IfdXMLi2luS05WjU4wnc2rv34GqxtnX9HhqTiVrmFa2FoCivFnMqryWpp7NHG17CYvJzfypHyPflv7B5YlmXbyQuM/H0Asvjz4MXlFG8Rc/j9YxehEYGxoC5WROaiTCwONPEh8ZRtHr0RcVUXjbp7AuXjgaoNEQ7e3F9z87iPv8aK1WDFWVlP7tlzGUlqTpwcRzTVlELOync8/LRIMeLK5ypq+7M7kYJewfTskpGvRw6Ll/S77vObSBnkMbsBVPY+aVXwZGHzOYdsln6Nj1PJ17/4rR7qJy6fW4p2Zu1HMq+9xFxAI+Bja8RNznwVhSTvktd6F7Z7oyNjKEckpOMa+H1gcfSL4fensDQ29vwFw9jco7/mY0xjNM1xOPkAj60VpsmKumUPm5r6GzZu4xAkU9cSd6AqT7F08+8pGPkJeXx29+8xu8Xi//+I//yBNPPEFfXx8lJSVcfPHF3HfffVRWVnLvvffy9NNPs2fPnpR2322/s+HxeHA6nVxi/Bg6Rf/uO5wnNHnZ+Rcqskkdee8rVz+o1FnTJrsLE67+9vN1QceZFexU3j3oPOOZkls5xcMhGn/8HUZGRnA4xr+XCxNc5D7IpMidP6TInR+kyJ0//jcXuQ/0v3gihBBCvB9S5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5SzfZHcg2bX4eWo1hsrsxYRJDw5PdhQmXCIUmuwsTb8+hye7BhKsoXzbZXciIjQ89NNldmHBXXXfLZHdhQsXiYRrPMlZGckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEztJNdgfOBy3+fTT5dhOJB7DrC5jlvJg8Q3HaWG90gAbvVkaifYTiXmY6LqTGtjAlptW/n1b/AYJxDwA2nYta+3IKTdWZTuVkH2JHaY4dIqIGsSn5zDIsw6kpGDe+O95CQ3QvIdWHRXEwXb+IQm158vNXgo+k3W+6bhFT9HMmvP/ptKkNtHCMCCFsOJnBIpyKK22sTx2hkUN4GSJEgDoWUKVMf19tZkqb2kCLevSdPuQxQzlzH3rUNhrVg4TwY8bGdGU+BUpp8vOwGqJB3ccAPcSIkk8BM5RFWBR7NtIBoLPpLToaNxIJe7E6Spk293rs+VVpY7tbttLbvhO/twcAm7OcmplXpcSrqkrr0Vfobt1GPBrE7qqhdt4NmG2FWckH4D9+PcxP/mOY7r44C2Yb+NmPClm+yDRu/PBInP/vx4M89YKPweE41RV6/u0HBVyzzgrA938ywA8eGErZZ8Y0PYc2Z+880da7jebut4hEfdgsJcysvBqnrTxtbGf/Hg42P5OyTaNoWbfk/0u+/+uO76fdd3rFZdSUrJ64jp9City76ArWc2RkM3Py1pKnL6HZv4cdA89yUdEtGLWWMfEJNYZZ66TEXMuRkc1p2zRpbcxwrMKiywNUOgJH2DX4PBcUfhy73p3ZhIDuWDNHozuZrV+BU+OmJXaEneHXWG36MEZl7B/lcLyP/ZHN1OoWUqitoDvexJ7IRlYar8GuyQNgjemjKfv0xzs5GH2bYm36E9dE61bbOMY+ZrEYBy7aqGc3m7hAvRJDmpzixLFgpZgKjrF3QtrMhG61jWPqXmYpi3Hgpk09xm71DS7gqrR9GFb7OaBuZZoyj0JK6VZb2au+yQoux6Y4UVWVfeqbKGhYoKxGh55W9Ri71DdYxZVolcyfEvo69tB06C/UzrsRe34VHcc3cWDrf7Pkkm9hMNrGxI8MNFJYvpCp+TVoNDraGzdwYMt/sXjtNzGanQB0NG6gs+lN6hZ9HJPFRcuRl0fbXPtNNFp9xnP60zNevnlvP//xr0WsWGTiZ78a5upPdnJ4cxVFBWN/ppGIypUf76SwQMtjvyqhvFRHS1uMPGfq5NqcGQZeeaws+V6nVTKeywndgwc42vYKs6o/hNNaQWvPFnbVP8LquV/BoLem3UenNXLB3K+M2+bFC76Z8r5/pJ5Dzc9SlD97Qvt+KpmufBfNvj1UWuZQYZmNTe9ijvMStIqOjsDhtPFOQzEznaspNdehKNq0MUWmKRSaarDq8rDq8qlzrEKn6BmJ9GQylaTm2GEqtLWU66Zh0+QxW78CLVo6Yw1p41viR3Brypiin4NN46RWvxCH4qItdjQZY1TMKa/eeBsuTQkWTXZGB60co5wplCk12BQHM1k8mhPNaeOdiovpynxKlEo04/wZnGubmdCqnujDlNE+KEvO2Ic2tR43JdQoM7AqDqZp5mInnzZ19NgG8DHCIDOVxTgVF1bFzkxlMXHidNOalZw6jm+ipGoFxVXLsNiLqZ1/I1qtnp7W7WnjZyz+FKU1F2BzlmGxFzF9wU2AynD/aE6qqtJxfDOVdetwl8zB6iilbtHHiYQ8DHQfzEpOP31wmM/f4uQzn3Awe4aBX95fiMWs8Os/eNPG/88fPAwOx3nq16WsXm6mplLPmgvMLJhjTInT6aCkSJd8FbjTn1MyoaVnCxUFiykvWITNXMis6mvRavR09O8+435GvS3ldabP+oaP4rJPwWLMz1geUuTOIKHG8UR7cRsrk9sURcFtrGA42j0h36GqCbqCx4ipUfIMJRPS5pkk1DhedRC39uT0laIouLSlDCf60+4zkujDrU3tm1tbynCiL218WA3Sn+igXDtt4jp+Bgk1gZdhXBQltymKgotihhn4wLT53vowhEs5OTWe7IOavg/DDOBSilK2uSlh5J0+qyQA0HDyZKkoCho0DKvpj/9ESiRi+EY6yCuoPeX7NeQVTMc71HJWbcTjEdREHL3BDEA4MEg07CWv4OR0s05vxp5Xiecs23w/IhGVnfvCrLvInNym0Sisu8jC2ztDaff5yyt+Vi4x8ZV7+iid18T8ta3c97NB4nE1Ja7+eJSKhU3Urmjm1i9309oezWguJyQScbz+TlyOqcltiqLgckxlxN8+7n7xeIRN+37KG3v/nT0Nf8QX7B03Nhz10T9ST1nBognt++lydroyHA4TDoeT7z0ezzm3EUkEUVExaM0p240aC/7I8Pvqnzfaz5b+J0ioMbSKnsWua7DpM3+vJ0J4NCdSp7qMigl/YiTtPmE1NCbeoJiIqOn/gDtjx9GipyhLU5XRcXIyYMTPuR/3TLU5cX0w4Sf9CCFCumNlTB4rC3ZMWGhQ9zOLJWjR0coxwgQJk/54TqRoxA9qAr0xdYSvN9oI+MY/IZ6q+dCLGEyOZFGLhEd/FqdPdRqMdqLh9D+nidQ/GCceh+LC1FFWcaGWow2RtPs0tUR5/c0Yn7rRxnOPlNLQHOUr9/QRjcF3vzl6Hli+yMT//KyYGdP0dPXE+OG/DbHmIx3s21CF3ZbZ8UkkFhj93TttWtKgs+IPpb8YspjczK65HrulmGg8REv322w/8j+smvNlTAbHmPiu/r1oNQaK8mdlJIcTcrbI3XfffXz/++lvcn4QWHX5XFD4cWKJCN2hBvYNv8oK941ZKXSZ1hFvpFQ7Be0407Vi8mgUDfO5gEPqdjaqz6Cg4KIIN5mfRZgIbfWv09+5h3kXfDEr99oyJaFCkVvLg/+nCK1WYckCE51dMX7yy+Fkkbt63ckCM3+2kRWLTUxZ1sJjz/r43KfGFo3JlmerJM92ctYrz1rJWwf/H+19O6gtv3RMfMfAbkrd89BqMluGcna68p577mFkZCT5amtrO+c2DBozCgqReDBlezgRSLvo5FxoFC1WXR5OQxEzHBfg0BXQ7E+/AGIiGTCO5nTaVXtYDWFUzGn3MSqmMfERNZR24cNQvJeA6qFCVzvms0zRj5NThPCYUc1ktjlxfRg7WjvBQLpjldpnh5LPSs0VrFU+wkXKdSzSXEyUCGbSLyaYSHqDFRTNmBFWNOzDYDzz/dv2xo20N7zOnJWfx+o4Od1+Yr9I2JcSHwl7x4wYM6HApUWrhZ6+eMr2nr44xUXpT+ClRVrqpunRnrKQZOZ0A929cSIRNe0+eU4tdVP1NDalHx1OJIPOMvq7F/WnbI/E/GPus41Ho9Fit5QSCA+N+WzI20IgNEB5weIJ6e8Z+5Hxb5gkRqMRh8OR8jpXGkWLQ1/EQORkgVRVlYFwO3n6ib3yVVFJqPF3D3yfNIoWu+JiIH7ynqKqqgzGu8kb5xECp6YwJR5gINFFnmbs8uyOeAMOxYVdk7kbyafTKBrs5DHIyekuVVUZpJc83ttq1Uy0+d76kM+gmqYPSvo+5OFOiQcYpAdnmj7rFD0GxUhA9eJhkEKlbEzMRNNodNic5clFIzB6X3q4vwF7/vhL49sbNtB2bD1zVn4Oe15lymdGiwu90c5wf31yWywawjvchuMMbU4Ug0FhyXwjr20+eTGcSKi8tjnAqiXpL0YuWGamoSlKInGyoNUfj1JarMVgSL+C0udP0NgSpbQ48xNwGo0Wu7WMQe/x5DZVVRn0HMdprTirNlQ1gS/Yk7YodvTvxm4pxW7J/AxCzha5iVJjW0i7/xAdgcP4ooMcHNlAXI1RbhmdR9439FeOet5Kxo8uVunDE+1DVeOE4n480T78seFkzFHPWwyGOwjEPHij/aPvIx2Umeuyk5NuFh3xejpijfgSIxyObiVOjDLd6EKR/ZE3qY+eXEFVrZ3JQKKT5ugh/IkRGqJ78SQGqdTNSGk3pkbojrdQnsVR3AlV1NFJE51qM37VwxF2ESdGKTUAHFC30aDuT8Yn1ARedRivOkyCBGGCeNVhAqrvrNvMSl5KHZ0cP9kH9bS8EttoSJzMq1KZzgDdtKhH8aseGhMH8TBIpXLymPSobQyqvQRUH71qB7vUNyikHLeSnSnL8qkX0d26jZ62HQS8PTTue4p4PEJx1VIAju7+I82HX0zGtze8TsvRl5m+4GZMZheRkJdIyEs8NnrPXVEUyqdeSFv9awx0H8Tv6eLY7j9hMDlwl2TnGc2vfyGP//q9h4cf83D4WIQv/30f/oDKHZ8YHUne/tUevvOjk/eyvni7g8HhOF//p36ONUZ4/lU/9/3fIb58hzMZ863v97PxrSDNbVHe2h7kxs92odXAJz6SnRXL1cUr6ejbRWf/HnzBPg63PEc8EaWsYCEAB5qeor791WR8Y+dGBkYaCYSH8Pi72N/0FKHwyJjRWiwepmfoUFZGcZDD9+QmSql5OpFEkHrvNsJxPw59IUvd1yWnK4NxL3DyyisU9/NW35+S75v9u2n27ybfUMaKghuB0QUt+4ZfJRz3o9cYsevcLHV9mAJTdhZqlOhqiBCmMbaPsBrEruSz2HhpcroypPpRTskpT1vIPMOFNET3UB/bg0Wxs9CwJvmM3And8dGVbCXamqzkcaoSpZKoGuY4hwgTwo6TRVyYfO4vRCAlpzBBtnLyD7SFY7RwjDwKWMras2oza3kR5rh68J0+5LFIuei0vE7KUwqYywoa1QM0cAALNhYoq7EpJ0+eYUIcU/cSIYQRM6VUM0XJ3HNKpyssX0g04qf16CvvPAxextwVn0tOO4aDwynHqqt5C2oizpGdv0tpp7LuMqpnXAFA+bS1xGMRGvY9QSwawuGqYe6Kz2Xtvt3Hr7fTPxDn3vsH6e6LsXCOkRceLaO4cPQU29YRRXPKkKKyXM+Lfyjjm9/rZ+E6D+UlWv72806+/ZWTMyDtXTFu+XI3A0NxCt1aVi8389bzlRQWZOded4lrLpFYgMbODYSjPuyWEhZPvyU5MguFRzj13BeLBTnU8hfCUR96rQmHtYxlsz6LzZw649M9eABQKXHNzUoeiqqq6SeAP+B+8Ytf8NRTT7F+/fqzivd4PDidTi4ruQudxpDh3mVPYmh4srsw4RKhzK/yyzolew/xZkvoQ8smuwsZsfGhhya7CxPuqutumewuTKhYPMzru3/MyMjIu96KOm+nK/v7+2lsbJzsbgghhPgAO2+L3L333ktzc/Nkd0MIIcQH2Hlb5IQQQoh3I0VOCCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5S4qcEEKInCVFTgghRM6SIieEECJnSZETQgiRs6TICSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ+kmuwPZFi91oWiNk92NCaNV1cnuwoRL9EYnuwsTTtFqJ7sLE07Jwd89gCd8jsnuwoRT4rl1rM4lHxnJCSGEyFlS5IQQQuQsKXJCCCFylhQ5IYQQOUuKnBBCiJwlRU4IIUTOkiInhBAiZ0mRE0IIkbOkyAkhhMhZUuSEEELkLClyQgghcpYUOSGEEDlLipwQQoicJUVOCCFEzpIiJ4QQImdJkRNCCJGzpMgJIYTIWVLkhBBC5CwpckIIIXKWFDkhhBA5SzfZHTgftPVuo7n7LSJRHzZLCTMrr8ZpKx83vmfwIA2drxMKD2Mxuaktv4zCvOnJz2PxCA3tr9I7fIRoLIjZmEdl0Qoqi5ZmIx0AWvz7afLvJhIPYNe7meW4mDxD8bjx3cEG6r1bCca9WHROZthXUWiqSX5e791Gd7CeUMKHghanvpDp9hXkGUqykM1JbYl6WtQjRAhhI48ZmsU4Ffe48T1qG42J/YTwY8bOdM18CpSy5OeNiQP0qK2ECKBBgwMX0zTzztjmRGuLH6M5fpgIQWxKPjO1S3BqCsaN70m00hDbRwgfFsVOrXYhhZqTv68xNUpDfA+9iXaiRDBjpVI7g0rt9HHbnGidzW/R3vgGkbAXm6OUaXOux55fmTa2q2UrvR27CHh7ALA5y6mZcVVKfDwWpunwiwz0HCQWCWC0uCifsprS6pVZyQfgpUf6+Mt/9TDcF6V6ppnPfreS2gXWtLFt9UH+9NMumg4G6OuIcPt3KvjQZ4reV5uZ0Nq3nebed8595mJmVVyN05r+3OcL9tLQtQFPsItQZIQZ5VdQXTT2538ubU6EcxrJrV27FkVRUBSFPXv2ZKhLH6w+dA8e4GjbK0wtW8OK2V/Abi5mV/0jRKL+tPHDvjb2H3+C8oJFrJj9BQrzZrC38Y/4gr3JmGNtL9PvaWDulBu5YO7fUFW8kqOtL9A7fDTj+QB0Bes54tlMrW0ZFxR8DLuugB2DfyEcD6SNH4p0sXf4lf+/vXuPjeq6Ezj+vXeeHs/LMzZ+P7CNAWNiQwATKAlJSLK07FZNSUsarSqlm2612l0p3UcLlZJVwlJV2iqJWvWPKInU1SbdrDY0aVOaJpDQjQKEphhjY8DYxu/3a2bsed979w/DmIEx2DBjh8n5SPcPX/3umfObx/nde869MkWW1WzJ/ga55nJOTfweX2QsFpOpd7LacS9bs/dQ7/4aGTobn43/lrASWJScAAbVblq105RLa9gkP4xNctKg/pGwFkwYP6mN0qwep0Aqp15+hGVSIY3qJ0xpk7GYTGyslNezWf4LNsgPYpYsnLpBm0nPSenignKKcl0N9Yad2CQnp6IfzZ2TOkJT9BMKdeXUG3aSIxXRGP2YKXU2p1blFKPqADX6LWwxfIUS3SouKJ8xrPYuSk4j/Y10tLxLSdWDrNv2j2Ta82k++Srh0FTCeM9YBzkFdazd/F1qt/4dJrODpk9fIRTwxGI6Wt5lYqSVlXV7uHv7P1G4/Eu0Nb/D2GDLouR07Hfj/OeBXnb/fT4/eXsVpasz+Pcn2/CMRRLGhwIqucVGvvXPBThzEl9rLLTNZBucOMuFvvepyLuPzSu/iy0jjz+3v05ojrFPUSNkmLJYUfAgRr01KW0mw4KnK5966ikGBgaoqamhs7MzVnCu3U6cOBE7JhAI8Oyzz1JVVYXJZCI7O5vHHnuMs2fPxrXt9/vZu3cvFRUVmM1mcnJyuO+++3jnnXdiMQcPHuTkyZO3kfLCdA2doCh7PYXZ67Bm5LC6dBc62UDfaEPC+O6hT3E7KinL24o1I4fKwgewW/LpHp7t8+RUDwXuWlz2MjJMTopy7sZqycM73bcoOXVOn6bYsoYiy2qsBhdrHNvRSXr6AucSxndNnyHbVMJy63qsBhcrbPXYDTl0+5tiMQUZVWSbirHoHdgMblbZv0RUC+OLji5KTgDd2gUKpXIK5HKskoNV0gZ06OnXLiWM79FacZNHmbyKTMlOhbwWG056tLZYTJ5cilvKwyJZsUoOqqR1KESYwpOwzWTrUs9TJFdQqKvAKjlYrduEDj19anvC+G71Am4pnzJdNVbJQaW+FruURbfaGouZ1EYp0C3HJeeSIVkp0lVilZx41bGEbSZbX8fH5BVvIq94I5m2XCrXfg1ZNjDU86eE8avWP05B2T1YHQVYrMtYUbsb0Jgcm/2cvBNd5Batx5ldgdniIr+0Hqs9H99kz6Lk9O5rwzz4zWzu3+2maEUGTz1XgjFD5qP/TfyeVt6VyV//sIitu1wYjImH4YW2mWydw8cpcq+n0F2HNSOH6uKvoJMN9I8lHvscmYWsLHyI/KwaZFmXlDaTYcFFzmKxkJeXh14/e/Zx+PBhBgYG4ra7774bgFAoxI4dO3jttdfYv38/ra2tHDp0iGg0Sn19fVwx/N73vsfBgwf52c9+xvnz53nvvffYvXs3Y2OzH6rL5SInJ+d2cp43VVXwTffjspfH9kmShMtejmc68VmvZ7onLh7Aba/AMzUb77QWMzLZSjDsRdM0xr2X8AfHcNsrUpPIVVRNwRsZwW0qiu2TJAm3qYjJ8GDCYybDg7hN8VNJ2abiOeNVTaHHfxa9ZMRmmHtaLZlUTcHHBC5pdspVkiRcUi6TWuJCO6mNxcUDuKV8PHPEq5pCn9aOHgNWnEnr+1xUTcGnjeOSZ6d8JUnCJefhURP30aOOxsXD9Tk5pWxG1D6Cmn/m+6cO4dd8uOX81CRyFVWN4vP04cyZnRqVJBlnTiXeie55taEoETRVQW+wxPbZs0oZGzpHKOBB0zQmR9sJTI2QlZP6KdhoWKXjrJ+1W2yxfbIssXaLjdaGW7tCSUWbC6GqCj7/AG7b8tg+SZJw2ZYz6b+1K/5UtDkfSVmTc7vd5OUlXnt58cUXOX78OA0NDdTW1gJQWlrKW2+9RX19Pd/5zndobm5GkiR+85vf8NJLL/HlL38ZgLKyslixXArhqB8NDaMhfg7cqM9kOph4kAlFpjDqr4k3WAlHZqdiVpXspKXrXT4+8wKSJAMS1aV/SZatNOk5XCusBmdyki1x+02yhenoRMJjQqofo5xxXXxIjZ/eHA520jj5BxQtiknOZKPrr647LlUihGfywhy334iZabwJjwkTTBBvIkz8VOCI1k+zehyFKCYyWCffh1EyJTeBhP0LLTinUKKcJDNhdTanVboNtCgn+TjyNhISIFGt20SWfP2aULJFwn7QVIym+Okso9FGYGpkXm10njuE0WwnK7sytq9izVe52PQWJ48cmPlNSRIr7vo6Dnf5DVpKDu9EFFUBZ3b8cOp06+lvv7Vp7VS0uRBhJfHYZ7rB2LcUbc5Hym88eeONN3jooYdiBe4KWZZ5+umneeKJJ2hsbKSuro68vDwOHTrEo48+is1mm6PF+QmFQoRCodjfXm/iQWEpdA+fxDPVS13lHsxGJxNTXZzvPoTJaMNtT/2PMlVcxkK2ZH+TiBqkx9/C6ck/sNm9G5POcvODP8dcLKNefpgIIfq0DprU42ySd2CUzDc/+HOoW23Fo45Sp78XM5lMaMOcVz7DJFlwy4t7o9BC9bR9xEh/I3fd87fIOkNsf3/nJ/gmuqne+G3MGVl4xi7R3vQ2RpN9Ua7mhM+vpDxCsGXLFqxWa9x2RWtrK6tXr0543JX9ra0z6wUvv/wyx44dw+12s3HjRp5++mk++eSTW+rTj3/8YxwOR2wrLk5859aNGPUWJKTrbjIJR6cxGRIvrJoMVsLRa+IjUxgvxytqhLa+I1QVP0yOcyU2Sy4lyzaR61pD1+CxBfdxoYyyeSana67CQqofk5y4GJlkC2E1cNN4vWwgU+/EacxjrfMBJGR651jnSzYDxpm8rrkKS3S1doURc4L40HXxOkmPRbLhkLKpljchIdGndSQ3gYT9M82Zk2mOnEyJctKCsYKsaFHalEaq9OvJkYuwyVmU6FaSK5fSpaT+szIYLSDJ191kEg77MJhufGLb2/5HetqOUlP/N2TaZ6dWFSVC5/k/UF69C3duNZn2fAqWbyG7oJa+jv9LSR5Xs2fpkXUwORqN2z85FsWZY5jjqMVvcyGMusRjX+gGY99StDkfSSlyb775JqdPn47brqZp2rzauffee+no6ODIkSPs3r2bs2fPsm3bNp5//vkF92nv3r14PJ7Y1tOz8AVoWdZhyyxg3Dc7oM2soXXgyCxKeIwjs5hxb/yNDmPeDhzWosvHq2iaiiRJcTESEhrze59uhyzpsBtyGAvNzoFrmsZYqHfO2/2dxry4eOCG8bF20VA15fY7PQ+ypMNGFuPa0Ozraxrj2hBOKfG6oFNyM64Nx+0b1wZxzBE/S0NFvd0u35Qs6bBJLsbVa3JSB+d8hMAhZzOuxq+Vjl2VkzbzqVyeppy1aN8/WY/NUcjk6OxNI5qmMjnahj2rZM7jetqO0n3xCDX1T2Jzxv/2NFVB0xS49jclSfMee26H3ihTvsZC83FfbJ+qajQf81G17tZu909FmwshyzpslnzGfLNjmaZpjPsu4bQkHvuWos15vW4yGikuLqaysjJuu6Kqqopz5xKfIV7ZX1VVFdtnMBjYtm0bP/jBD3j//fd57rnneP755wmHwwvqk8lkwm63x223ojR3M30jp+gfPc1UYIRzXe+iqBEKsusAaL70ay72Ho7Fl+TWM+Zto3PwGNOBUdr7juL191OybBMAep2JLGsprT0fMO7tJBCaoH/0NANjZ1jmXHVLfVyossw6ev0t9PnPMxUZ56z3KIoWpTBj5sr6zORhLniPz74HmXcxGurm0lQDU9EJLvpO4okMU2JZC0BUjdDqPc5keJBA1IsnMkzT5BFCyjR55tTfTHNFibSSfq2DfvUS05qX89pnKETJl2YWupvVE7SpZ2LxxVIVYwzQpZ5nWvPSrjbjZYJiaeb7q2hR2tQzeLRRAto0Xm2cs+pJQgTIlRY+M3ArSuVV9Klt9CsdTGkezil/QiFKgTwzrd0cPcbF6OnZ90BeyZg2QKdyjmnNQ3v0DF5tnBJ55jemlwxkSctoVRoYV4cIaFP0Kx0MqJdYJi9OToXl2xjsPslQz5/x+4Zoa/o1qhIht3jmOdELDW9y6dzvY/E9bUfpan2fqtrHMGe4CAd9hIM+lOjMcoTeYMbhKufSuUNMjrYT9I8z1PMZw72ncOetWZScdj25jCNvjnL04Bi9bQFeeaaHUEBl+9dnnqf8+b908sZ/zN49HQ2rdLb46WzxE41ojA+F6WzxM9gVnHebqVa27B76xk7RN9bIVHCEcz2/mxn73HUANHW+zcX+I7F4VVXw+gfx+gfRVIVgxIfXP4g/ND7vNlMh5Wtye/bs4Uc/+hGNjY1x63KqqvLCCy9QXV193Xrd1aqrq4lGowSDQYxGY6q7e508Vw3hqJ/2/qOEIlPYLHmsX/FE7PI6GPLAVWfFTmsxa5c/SlvfR7T1fYjF5KK2Yg/WjNlF/bUVu2nrPULzpYNEogHMJgeVhQ9QlLM4D4PnZ6wgrAa4OPUpIcWP3ZDNBteu2NpZQPHF5ZRlzKfW+RCtvk9p9Z0gU+9kfdZObIaZH5skSUxHJ2mYeI+wGsAom3EYllHv/losZjHkySVE1BAdWjMhLYgNJ+vk+zBdnqoLav64K2inlE2NfA/tahNtWhMWrNTKW7FKzssREtOalwGtkzAhDBix4+Ju+QGskmNxctKVEiZIu3KGkBLEJmWxXn8/JikjltPVVzBOOYe1+q20RRtpUxqxSDZq9duwys5YzFr9VtqURpqjx4gQxkwmlbq7KJIrr335lMgpqCUSmqar9f3LD4MXsGbTkxgvT1eGApNxOQ10nUBTFc79+b/i2ilZsYPSlQ8BsGr9t+g8/3suNPw30YgfU0YWpaseWbSHwbd8xYV3PMr/vDTA5EiEstUZ7Hu1Emf2zNTiaH847kJzfDjCv371fOzv3746zG9fHaZ6k5V/e71qXm2mWl7WGsLRadoHjhKKTmHLyGV9xbdmx76IJ+73FIr4OHHh5djfXcPH6Ro+Tpa1lI0rvj2vNlNB0hZwPb99+3bq6up48cUXAejs7GT58uUcPnyYNWviz5icTidms5lgMMj27dvp7+/npz/9KfX19QwNDXHgwAE++OADDh8+zObNm2PtP/7442zYsAG3201LSwvf//73KSws5MiR2TOGK6/b0NBAXV3dvPru9XpxOBzcv+6H6HWpvzNusej6Fu85tMUSHU6/nCRd4ueG7mTBh+c+Ob2T/cMLby51F5LulW/sWuouJFVUCfHhmZ/g8XhuOkuXlCu5HTt2XLfvV7/6FXv27MFsNvPhhx9y4MAB9u3bR1dXFzabjfvvv58TJ05QU1MTO+aRRx7hl7/8Jfv27cPv91NQUMCuXbt45plnktFNQRAE4QvmtopcWVnZvBZ2LRYL+/fvZ//+/TeM27t3L3v37r2dLgmCIAhCzIJvPPnFL36B1Wqlqanp5sEpsHPnzuumRgVBEAQhkQVdyb3++usEAjPPS5WUzH27byq98sorS94HQRAE4c6woCJXWJi6f4dwJ/VBEARBuDOIf5oqCIIgpC1R5ARBEIS0JYqcIAiCkLZEkRMEQRDSlihygiAIQtoSRU4QBEFIW6LICYIgCGlLFDlBEAQhbYkiJwiCIKQtUeQEQRCEtCWKnCAIgpC2RJETBEEQ0pYocoIgCELaEkVOEARBSFuiyAmCIAhpa0H/T+5OpmkaAFEltMQ9SS5NDS91F5IuqkWWugtJJ2nqUnch6aKR4FJ3ISX8PmWpu5B06TbuXcnnyrh+I5I2n6g00NvbS3Fx8VJ3QxAEQUiSnp4eioqKbhjzhSlyqqrS39+PzWZDkqSUvpbX66W4uJienh7sdntKX2uxiJzuDCKnO0M65gSLl5emafh8PgoKCpDlG6+6fWGmK2VZvmnFTza73Z5WX2AQOd0pRE53hnTMCRYnL4fDMa84ceOJIAiCkLZEkRMEQRDSlihyKWAymXj22WcxmUxL3ZWkETndGUROd4Z0zAk+n3l9YW48EQRBEL54xJWcIAiCkLZEkRMEQRDSlihygiAIQtoSRU4QBEFIW6LICYIgCGlLFDlBEAQhbYkiJwiCIKQtUeQEQRCEtPX/ahRhUsC/UogAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 480x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "it s very cold here .\n"
     ]
    }
   ],
   "source": [
    "translator = Translator(model.cpu(), src_tokenizer, trg_tokenizer)\n",
    "result = translator(u'hace mucho frio aqui .')\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8570b29725840556",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T04:37:15.037460Z",
     "start_time": "2025-01-27T04:37:15.037460Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:11:39.915710Z",
     "iopub.status.busy": "2025-01-27T13:11:39.915421Z",
     "iopub.status.idle": "2025-01-27T13:11:40.249024Z",
     "shell.execute_reply": "2025-01-27T13:11:40.248406Z",
     "shell.execute_reply.started": "2025-01-27T13:11:39.915688Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx))\n",
    "model.load_state_dict(torch.load(\"checkpoints/01_best.ckpt\",weights_only=True, map_location=device))\n",
    "\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.model = model\n",
    "        self.model.eval()  # 切换到验证模式\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "\n",
    "    def __call__(self, sentence):\n",
    "        # 预处理句子，标点符号处理等\n",
    "        sentence = preprocess_sentence(sentence)\n",
    "        encoder_inputs, attn_mask = self.src_tokenizer.encode(\n",
    "            [sentence.split()],\n",
    "            padding_first=True,\n",
    "            add_bos=True,\n",
    "            add_eos=True,\n",
    "            return_mask=True,\n",
    "        )  # 对输入进行编码，并返回encoder_inputs和encoder_padding_mask\n",
    "        # 转换成tensor\n",
    "        encoder_inputs = torch.Tensor(encoder_inputs).to(dtype=torch.int64)\n",
    "        # 由输入得到预测结果，\n",
    "        # 注意，这里输入的样本只有一个，符合Seq2Seq模型的生成方法要求batch_size=1\n",
    "        preds, scores = self.model.infer(encoder_inputs=encoder_inputs, attn_mask=attn_mask)\n",
    "\n",
    "        # 通过tokenizer转换成文字\n",
    "        # trg_tokenizer.decode方法要求输入的是字符串列表，所以需要[preds]\n",
    "        # 方法返回字符串列表，所以需要[0]\n",
    "        trg_sentence = self.trg_tokenizer.decode([preds], split=True, remove_eos=True)[0]\n",
    "        \n",
    "        # 为了移除句子末尾的特殊标记\n",
    "        return \" \".join(trg_sentence[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "18b2c2345791f9d5",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T13:11:40.250246Z",
     "iopub.status.busy": "2025-01-27T13:11:40.249717Z",
     "iopub.status.idle": "2025-01-27T13:17:56.036876Z",
     "shell.execute_reply": "2025-01-27T13:17:56.036186Z",
     "shell.execute_reply.started": "2025-01-27T13:11:40.250222Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 2-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 3-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time used: 375.3174800872803\n"
     ]
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "# from torchmetrics.text.bleu import BLEUScore\n",
    "\n",
    "# BLEU 分数的计算原理\n",
    "# n-gram 精度：\n",
    "# BLEU 基于 n-gram（连续的 n 个单词）的匹配程度。\n",
    "# 例如，1-gram 是单个单词，2-gram 是两个连续的单词，依此类推。\n",
    "# 计算候选翻译中每个 n-gram 在参考翻译中出现的次数，并除以候选翻译中该 n-gram 的总数。\n",
    "\n",
    "def evaluate_bleu_on_test_set(test_data, translator):\n",
    "    # 在测试集上计算平均 BLEU 分数。\n",
    "    total_bleu = 0\n",
    "    num_samples = len(test_data)\n",
    "    for src_sentence, ref_translation in test_data:\n",
    "        # 使用翻译器生成翻译结果\n",
    "        candidate_translation = translator(src_sentence)\n",
    "\n",
    "        # 计算 BLEU 分数\n",
    "        bleu_score = sentence_bleu(\n",
    "            [ref_translation.split()], candidate_translation.split(),\n",
    "            weights=(1, 0, 0, 0)\n",
    "        )\n",
    "        total_bleu += bleu_score\n",
    "    avg_bleu = total_bleu / num_samples\n",
    "    return avg_bleu\n",
    "\n",
    "translator=Translator(model, src_tokenizer, trg_tokenizer)\n",
    "\n",
    "start_time = time.time()\n",
    "avg_bleu=evaluate_bleu_on_test_set(test_ds, translator)\n",
    "end_time = time.time()\n",
    "print(\"Time used:\", end_time - start_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "460dd740-6aec-4398-a6c6-71ca45ac206a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T13:17:56.037890Z",
     "iopub.status.busy": "2025-01-27T13:17:56.037533Z",
     "iopub.status.idle": "2025-01-27T13:17:56.040742Z",
     "shell.execute_reply": "2025-01-27T13:17:56.040296Z",
     "shell.execute_reply.started": "2025-01-27T13:17:56.037869Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg_bleu:0.5507854547137234\n"
     ]
    }
   ],
   "source": [
    "print(f\"avg_bleu:{avg_bleu}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
